1 | /* |
2 | * Copyright (c) 2015-2023 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <sys/param.h> |
30 | #include <sys/sdt.h> |
31 | #include <sys/systm.h> |
32 | #include <sys/filedesc.h> |
33 | #include <sys/proc_internal.h> |
34 | #include <sys/file_internal.h> |
35 | #include <sys/vnode_internal.h> |
36 | #include <sys/sysproto.h> |
37 | #include <security/audit/audit.h> |
38 | #include <skywalk/os_skywalk_private.h> |
39 | |
40 | static int chop_select(struct fileproc *, int, void *, vfs_context_t); |
41 | static int chop_close(struct fileglob *, vfs_context_t); |
42 | static int chop_kqfilter(struct fileproc *, struct knote *, struct kevent_qos_s *); |
43 | |
44 | static const struct fileops channel_ops = { |
45 | .fo_type = DTYPE_CHANNEL, |
46 | .fo_read = fo_no_read, |
47 | .fo_write = fo_no_write, |
48 | .fo_ioctl = fo_no_ioctl, |
49 | .fo_select = chop_select, |
50 | .fo_close = chop_close, |
51 | .fo_drain = fo_no_drain, |
52 | .fo_kqfilter = chop_kqfilter, |
53 | }; |
54 | |
55 | #if (DEVELOPMENT || DEBUG) |
56 | static uint32_t ch_force_defunct = 0; |
57 | SYSCTL_UINT(_kern_skywalk_channel, OID_AUTO, force_defunct, |
58 | CTLFLAG_RW | CTLFLAG_LOCKED, &ch_force_defunct, 0, "" ); |
59 | #endif /* !DEVELOPMENT && !DEBUG */ |
60 | |
61 | static int |
62 | chop_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx) |
63 | { |
64 | int revents = 0, events = 0; |
65 | struct kern_channel *ch; |
66 | |
67 | ch = (struct kern_channel *)fp_get_data(fp); |
68 | if (ch == NULL) { |
69 | return 0; |
70 | } |
71 | |
72 | switch (which) { |
73 | case FREAD: |
74 | events = POLLIN; |
75 | break; |
76 | |
77 | case FWRITE: |
78 | events = POLLOUT; |
79 | break; |
80 | |
81 | case 0: |
82 | events = POLLERR; |
83 | break; |
84 | |
85 | default: |
86 | return 1; |
87 | } |
88 | |
89 | /* caller will lock channel */ |
90 | revents = ch_select(ch, events, wql, vfs_context_proc(ctx)); |
91 | return (events & revents) ? 1 : 0; |
92 | } |
93 | |
94 | static int |
95 | chop_close(struct fileglob *fg, vfs_context_t ctx) |
96 | { |
97 | #pragma unused(ctx) |
98 | struct kern_channel *ch; |
99 | |
100 | ch = (struct kern_channel *)fg_get_data(fg); |
101 | fg_set_data(fg, NULL); |
102 | if (ch != NULL) { |
103 | ch_dtor(ch); |
104 | } |
105 | |
106 | return 0; |
107 | } |
108 | |
109 | /* |
110 | * This is called when a thread invokes kevent() to record |
111 | * a change in the configuration of the kqueue(). |
112 | */ |
113 | static int |
114 | chop_kqfilter(struct fileproc *fp, struct knote *kn, struct kevent_qos_s *kev) |
115 | { |
116 | struct kern_channel *ch = (struct kern_channel *)fp_get_data(fp); |
117 | |
118 | /* caller will lock channel */ |
119 | return ch_kqfilter(ch, kn, kev); |
120 | } |
121 | |
122 | int |
123 | __channel_open(struct proc *p, struct __channel_open_args *uap, int *retval) |
124 | { |
125 | struct fileproc *fp = NULL; |
126 | struct kern_channel *ch = NULL; |
127 | struct ch_init init; |
128 | int fd = -1, err = 0; |
129 | void *key = NULL; |
130 | uint32_t keylen; |
131 | guardid_t guard; |
132 | |
133 | if (__improbable(uap->init == USER_ADDR_NULL || |
134 | uap->init_len < sizeof(init))) { |
135 | SK_DSC(p, "EINVAL: init 0x%llx, init_len %u" , SK_KVA(uap->init), |
136 | uap->init_len); |
137 | err = EINVAL; |
138 | goto done; |
139 | } |
140 | |
141 | err = copyin(uap->init, (caddr_t)&init, sizeof(init)); |
142 | if (__improbable(err != 0)) { |
143 | SK_DSC(p, "copyin err %u: init 0x%llx" , err, SK_KVA(uap->init)); |
144 | goto done; |
145 | } |
146 | |
147 | if (__improbable(init.ci_version != CHANNEL_INIT_CURRENT_VERSION)) { |
148 | SK_DSC(p, "ENOTSUP: init.ci_version %u != %u" , init.ci_version, |
149 | CHANNEL_INIT_CURRENT_VERSION); |
150 | err = ENOTSUP; |
151 | goto done; |
152 | } else if (__improbable(uuid_is_null(init.ci_nx_uuid))) { |
153 | SK_DSC(p, "EINVAL: uuid_is_null" ); |
154 | err = EINVAL; |
155 | goto done; |
156 | } else if (__improbable((init.ci_key_len != 0 && |
157 | init.ci_key == USER_ADDR_NULL) || |
158 | (init.ci_key_len == 0 && init.ci_key != USER_ADDR_NULL))) { |
159 | SK_DSC(p, "EINVAL: ci_key_len %i, ci_key 0x%llx" , |
160 | init.ci_key_len, SK_KVA(init.ci_key)); |
161 | err = EINVAL; |
162 | goto done; |
163 | } |
164 | |
165 | if ((init.ci_ch_mode & CHMODE_MONITOR) != 0) { |
166 | if (__improbable((init.ci_ch_mode & CHMODE_USER_PACKET_POOL) != 0)) { |
167 | SK_DSC(p, "EINVAL: PACKET_POOL not supported for MONITOR mode" ); |
168 | err = EINVAL; |
169 | goto done; |
170 | } |
171 | if (__improbable((init.ci_ch_mode & CHMODE_EVENT_RING) != 0)) { |
172 | SK_DSC(p, "EINVAL: EVENT ring not supported for MONITOR mode" ); |
173 | err = EINVAL; |
174 | goto done; |
175 | } |
176 | if (__improbable((init.ci_ch_mode & CHMODE_LOW_LATENCY) != 0)) { |
177 | SK_DSC(p, "EINVAL: low latency not supported for MONITOR mode" ); |
178 | err = EINVAL; |
179 | goto done; |
180 | } |
181 | } |
182 | |
183 | if ((init.ci_ch_mode & CHMODE_EVENT_RING) != 0) { |
184 | if ((init.ci_ch_mode & CHMODE_USER_PACKET_POOL) == 0) { |
185 | SK_DSC(p, "EINVAL: PACKET_POOL is required for EVENT ring" ); |
186 | err = EINVAL; |
187 | goto done; |
188 | } |
189 | } |
190 | |
191 | #if (DEVELOPMENT || DEBUG) |
192 | if (__improbable(ch_force_defunct)) { |
193 | init.ci_ch_mode |= CHMODE_DEFUNCT_OK; |
194 | } |
195 | #endif /* !DEVELOPMENT && !DEBUG */ |
196 | |
197 | /* generate guard ID based on nexus instance UUID */ |
198 | sk_gen_guard_id(TRUE, init.ci_nx_uuid, &guard); |
199 | |
200 | err = falloc_guarded(p, fp: &fp, fd: &fd, ctx: vfs_context_current(), guard: &guard, |
201 | GUARD_CLOSE | GUARD_DUP | GUARD_SOCKET_IPC | GUARD_FILEPORT | GUARD_WRITE); |
202 | if (__improbable(err != 0)) { |
203 | SK_DSC(p, "falloc_guarded: %u" , err); |
204 | goto done; |
205 | } |
206 | |
207 | keylen = init.ci_key_len; |
208 | if (keylen != 0) { |
209 | if (__improbable(keylen > NEXUS_MAX_KEY_LEN)) { |
210 | SK_DSC(p, "EINVAL: ci_key_len %u" , keylen); |
211 | err = EINVAL; |
212 | goto done; |
213 | } |
214 | |
215 | key = sk_alloc_data(keylen, Z_WAITOK, skmem_tag_ch_key); |
216 | if (__improbable(key == NULL)) { |
217 | SK_DSC(p, "ENOMEM: ci_key_len %u" , keylen); |
218 | err = ENOMEM; |
219 | goto done; |
220 | } |
221 | |
222 | err = copyin(init.ci_key, (caddr_t)key, keylen); |
223 | if (__improbable(err != 0)) { |
224 | SK_DSC(p, "copyin err %u: ci_key 0x%llx, ci_key_len %u" , |
225 | err, SK_KVA(init.ci_key), keylen); |
226 | goto done; |
227 | } |
228 | } |
229 | |
230 | /* let ch_open() take over this key upon success */ |
231 | init.ci_key = (user_addr_t)key; |
232 | key = NULL; |
233 | |
234 | if (__improbable((ch = ch_open(&init, p, fd, &err)) == NULL)) { |
235 | /* in case not processed */ |
236 | key = (void *)init.ci_key; |
237 | ASSERT(err != 0); |
238 | SK_DSC(p, "ch_open nx_port %d err %u" , |
239 | (int)init.ci_nx_port, err); |
240 | goto done; |
241 | } |
242 | /* in case not processed */ |
243 | key = (void *)init.ci_key; |
244 | |
245 | /* update userland with respect to guard ID, etc. */ |
246 | init.ci_guard = guard; |
247 | init.ci_key = USER_ADDR_NULL; |
248 | err = copyout(&init, uap->init, sizeof(init)); |
249 | if (__improbable(err != 0)) { |
250 | SK_DSC(p, "copyout err %u: init 0x%llx" , err, |
251 | SK_KVA(uap->init)); |
252 | goto done; |
253 | } |
254 | |
255 | fp->fp_flags |= FP_CLOEXEC | FP_CLOFORK; |
256 | fp->fp_glob->fg_flag &= ~(FREAD | FWRITE); |
257 | fp->fp_glob->fg_ops = &channel_ops; |
258 | fp_set_data(fp, fg_data: ch); /* ref from ch_open */ |
259 | |
260 | proc_fdlock(p); |
261 | procfdtbl_releasefd(p, fd, NULL); |
262 | fp_drop(p, fd, fp, locked: 1); |
263 | proc_fdunlock(p); |
264 | |
265 | *retval = fd; |
266 | |
267 | SK_D("%s(%d) nx_port %d fd %d guard 0x%llx" , |
268 | sk_proc_name_address(p), sk_proc_pid(p), (int)init.ci_nx_port, |
269 | fd, guard); |
270 | |
271 | done: |
272 | if (key != NULL) { |
273 | sk_free_data(key, keylen); |
274 | key = NULL; |
275 | } |
276 | if (__improbable(err != 0)) { |
277 | if (ch != NULL) { |
278 | ch_dtor(ch); |
279 | ch = NULL; |
280 | } |
281 | if (fp != NULL) { |
282 | fp_free(p, fd, fp); |
283 | fp = NULL; |
284 | } |
285 | } |
286 | |
287 | return err; |
288 | } |
289 | |
290 | int |
291 | __channel_get_info(struct proc *p, struct __channel_get_info_args *uap, |
292 | int *retval) |
293 | { |
294 | #pragma unused(retval) |
295 | struct fileproc *fp; |
296 | struct kern_channel *ch = NULL; |
297 | int err = 0; |
298 | |
299 | AUDIT_ARG(fd, uap->c); |
300 | |
301 | err = fp_get_ftype(p, fd: uap->c, ftype: DTYPE_CHANNEL, ENODEV, fpp: &fp); |
302 | if (__improbable(err != 0)) { |
303 | SK_DSC(p, "fp_get_ftype err %u" , err); |
304 | return err; |
305 | } |
306 | ch = fp_get_data(fp); |
307 | |
308 | if (__improbable(uap->cinfo == USER_ADDR_NULL || |
309 | uap->cinfolen < sizeof(struct ch_info))) { |
310 | SK_DSC(p, "EINVAL: cinfo 0x%llx, cinfolen %u" , |
311 | SK_KVA(uap->cinfo), uap->cinfolen); |
312 | err = EINVAL; |
313 | goto done; |
314 | } |
315 | |
316 | lck_mtx_lock(lck: &ch->ch_lock); |
317 | err = copyout(ch->ch_info, uap->cinfo, sizeof(struct ch_info)); |
318 | lck_mtx_unlock(lck: &ch->ch_lock); |
319 | if (__improbable(err != 0)) { |
320 | SK_DSC(p, "copyout err %u: cinfo 0x%llx" , err, |
321 | SK_KVA(uap->cinfo)); |
322 | goto done; |
323 | } |
324 | |
325 | done: |
326 | fp_drop(p, fd: uap->c, fp, locked: 0); |
327 | |
328 | return err; |
329 | } |
330 | |
331 | #if SK_LOG |
332 | /* Hoisted out of line to reduce kernel stack footprint */ |
333 | SK_LOG_ATTRIBUTE |
334 | static void |
335 | channel_sync_log1(uint64_t verb, const char *sync, struct proc *p, |
336 | const struct nexus_adapter *na, const struct kern_channel *ch, |
337 | const struct __kern_channel_ring *kring, ring_id_t i) |
338 | { |
339 | verb |= SK_VERB_SYNC; |
340 | SK_DF(verb, "%s(%d) pre: %s ring %u na \"%s\" (0x%llx) ch 0x%llx " |
341 | "th 0x%llx h %u kh %u" , sk_proc_name_address(p), sk_proc_pid(p), |
342 | sync, i, na->na_name, SK_KVA(na), SK_KVA(ch), |
343 | SK_KVA(current_thread()), kring->ckr_ring->ring_head, |
344 | kring->ckr_khead); |
345 | } |
346 | |
347 | SK_LOG_ATTRIBUTE |
348 | static void |
349 | channel_sync_log2(uint64_t verb, const char *sync, struct proc *p, |
350 | const struct nexus_adapter *na, const struct __kern_channel_ring *kring, |
351 | ring_id_t i) |
352 | { |
353 | verb |= SK_VERB_SYNC; |
354 | SK_DF(verb, "%s(%d) post: %s ring %u na \"%s\" h %u kh %u" , |
355 | sk_proc_name_address(p), sk_proc_pid(p), sync, i, na->na_name, |
356 | kring->ckr_ring->ring_head, kring->ckr_khead); |
357 | } |
358 | #endif /* SK_LOG */ |
359 | |
360 | int |
361 | __channel_sync(struct proc *p, struct __channel_sync_args *uap, int *retval) |
362 | { |
363 | #pragma unused(retval) |
364 | struct fileproc *fp; |
365 | sk_protect_t protect = NULL; |
366 | struct nexus_adapter *na; |
367 | struct __kern_channel_ring *krings, *kring; |
368 | struct kern_channel *ch; |
369 | sync_mode_t mode; |
370 | ring_id_t i, qfirst, qlast; |
371 | sync_flags_t flags, upp_sync_flags = 0; |
372 | enum txrx t; |
373 | int err; |
374 | int s; |
375 | |
376 | net_update_uptime(); |
377 | |
378 | AUDIT_ARG(fd, uap->c); |
379 | |
380 | err = fp_get_ftype(p, fd: uap->c, ftype: DTYPE_CHANNEL, ENODEV, fpp: &fp); |
381 | if (__improbable(err != 0)) { |
382 | SK_DSC(p, "fp_get_ftype err %u" , err); |
383 | return err; |
384 | } |
385 | ch = fp_get_data(fp); |
386 | |
387 | lck_mtx_lock(lck: &ch->ch_lock); |
388 | ASSERT(!(ch->ch_flags & CHANF_KERNEL)); |
389 | |
390 | mode = uap->mode; |
391 | flags = uap->flags; |
392 | if (__improbable(mode != CHANNEL_SYNC_TX && mode != CHANNEL_SYNC_RX && |
393 | mode != CHANNEL_SYNC_UPP)) { |
394 | SK_DSC(p, "EINVAL: mode %u" , mode); |
395 | err = EINVAL; |
396 | goto done; |
397 | } |
398 | |
399 | if (__improbable((ch->ch_flags & CHANF_USER_PACKET_POOL) == 0 && |
400 | (flags & (CHANNEL_SYNCF_ALLOC | CHANNEL_SYNCF_FREE | |
401 | CHANNEL_SYNCF_ALLOC_BUF)) != 0)) { |
402 | SK_DSC(p, "EINVAL: !CHANF_USER_PACKET_POOL with " |
403 | "SYNCF_ALLOC/FREE" ); |
404 | err = EINVAL; |
405 | goto done; |
406 | } |
407 | |
408 | if (__improbable(ch->ch_flags & CHANF_DEFUNCT)) { |
409 | SK_DSC(p, "channel is defunct" ); |
410 | err = ENXIO; |
411 | goto done; |
412 | } |
413 | |
414 | /* clear CHANF_DEFUNCT_SKIP if it was set during defunct last time */ |
415 | if (__improbable(ch->ch_flags & CHANF_DEFUNCT_SKIP)) { |
416 | os_atomic_andnot(&ch->ch_flags, CHANF_DEFUNCT_SKIP, relaxed); |
417 | } |
418 | |
419 | na = ch->ch_na; /* we have a reference */ |
420 | ASSERT(na != NULL); |
421 | ASSERT(NA_IS_ACTIVE(na)); |
422 | |
423 | if (__improbable(na_reject_channel(ch, na))) { |
424 | SK_DSC(p, "channel is non-permissive" ); |
425 | err = ENXIO; |
426 | goto done; |
427 | } |
428 | |
429 | /* mark thread with sync-in-progress flag */ |
430 | protect = sk_sync_protect(); |
431 | |
432 | /* update our work timestamp */ |
433 | na->na_work_ts = _net_uptime; |
434 | |
435 | /* and make this channel eligible for draining again */ |
436 | if (na->na_flags & NAF_DRAINING) { |
437 | os_atomic_andnot(&na->na_flags, NAF_DRAINING, relaxed); |
438 | } |
439 | |
440 | if (mode == CHANNEL_SYNC_UPP) { |
441 | upp_sync_flags = NA_SYNCF_FORCE_UPP_SYNC; |
442 | if (flags & CHANNEL_SYNCF_PURGE) { |
443 | upp_sync_flags |= NA_SYNCF_UPP_PURGE; |
444 | } |
445 | goto packet_pool_sync; |
446 | } |
447 | |
448 | t = (mode == CHANNEL_SYNC_TX ? NR_TX : NR_RX); |
449 | krings = NAKR(na, t); |
450 | qfirst = ch->ch_first[t]; |
451 | qlast = ch->ch_last[t]; |
452 | |
453 | for (i = qfirst; i < qlast; i++) { |
454 | kring = krings + i; |
455 | s = kr_enter(kring, TRUE); |
456 | ASSERT(s == 0); |
457 | #if SK_LOG |
458 | if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) { |
459 | channel_sync_log1((mode == CHANNEL_SYNC_TX) ? |
460 | SK_VERB_TX : SK_VERB_RX, (mode == CHANNEL_SYNC_TX) ? |
461 | "txsync" : "rxsync" , p, na, ch, kring, i); |
462 | } |
463 | #endif /* SK_LOG */ |
464 | |
465 | switch (mode) { |
466 | case CHANNEL_SYNC_TX: { |
467 | int error = 0; |
468 | |
469 | DTRACE_SKYWALK2(pretxprologue, struct kern_channel *, |
470 | ch, struct __kern_channel_ring *, kring); |
471 | |
472 | if (kr_txsync_prologue(ch, kring, p) >= |
473 | kring->ckr_num_slots) { |
474 | kr_log_bad_ring(kring); |
475 | error = EFAULT; |
476 | if (!err) { |
477 | SK_DSC(p, "EFAULT: " |
478 | "kr_txsync_prologue()" ); |
479 | err = EFAULT; |
480 | } |
481 | } else if (kring->ckr_na_sync(kring, p, |
482 | NA_SYNCF_FORCE_RECLAIM) == 0) { |
483 | kr_txsync_finalize(ch, kring, p); |
484 | } else { |
485 | error = EIO; |
486 | if (!err) { |
487 | SK_DSC(p, "EIO: TX " |
488 | "kring->ckr_na_sync()" ); |
489 | err = EIO; |
490 | } |
491 | } |
492 | |
493 | DTRACE_SKYWALK3(posttxfinalize, struct kern_channel *, |
494 | ch, struct __kern_channel_ring *, kring, int, |
495 | error); |
496 | } |
497 | break; |
498 | |
499 | case CHANNEL_SYNC_RX: { |
500 | int error = 0; |
501 | |
502 | DTRACE_SKYWALK2(prerxprologue, struct kern_channel *, |
503 | ch, struct __kern_channel_ring *, kring); |
504 | |
505 | if (kr_rxsync_prologue(ch, kring, p) >= |
506 | kring->ckr_num_slots) { |
507 | kr_log_bad_ring(kring); |
508 | error = EFAULT; |
509 | if (!err) { |
510 | SK_DSC(p, "EFAULT: " |
511 | "kr_rxsync_prologue()" ); |
512 | err = EFAULT; |
513 | } |
514 | } else if (kring->ckr_na_sync(kring, p, |
515 | NA_SYNCF_FORCE_READ) == 0) { |
516 | kr_rxsync_finalize(ch, kring, p); |
517 | } else { |
518 | error = EIO; |
519 | if (!err) { |
520 | SK_DSC(p, "EIO: " "RX " |
521 | "kring->ckr_na_sync()" ); |
522 | err = EIO; |
523 | } |
524 | } |
525 | |
526 | DTRACE_SKYWALK3(postrxfinalize, struct kern_channel *, |
527 | ch, struct __kern_channel_ring *, kring, int, |
528 | error); |
529 | } |
530 | break; |
531 | |
532 | default: |
533 | VERIFY(0); |
534 | /* NOTREACHED */ |
535 | __builtin_unreachable(); |
536 | } |
537 | |
538 | #if SK_LOG |
539 | if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) { |
540 | channel_sync_log2((mode == CHANNEL_SYNC_TX) ? |
541 | SK_VERB_TX : SK_VERB_RX, (mode == CHANNEL_SYNC_TX) ? |
542 | "txsync" : "rxsync" , p, na, kring, i); |
543 | } |
544 | #endif /* SK_LOG */ |
545 | |
546 | kr_exit(kring); |
547 | } |
548 | |
549 | packet_pool_sync: |
550 | if (flags & (CHANNEL_SYNCF_ALLOC | |
551 | CHANNEL_SYNCF_LARGE_ALLOC | CHANNEL_SYNCF_ALLOC_BUF)) { |
552 | enum txrx type; |
553 | |
554 | if (flags & CHANNEL_SYNCF_LARGE_ALLOC) { |
555 | ASSERT(!(flags & CHANNEL_SYNCF_ALLOC)); |
556 | ASSERT(!(flags & CHANNEL_SYNCF_ALLOC_BUF)); |
557 | type = NR_LBA; |
558 | qfirst = ch->ch_first[type]; |
559 | qlast = ch->ch_last[type]; |
560 | } else { |
561 | type = NR_A; |
562 | qfirst = ch->ch_first[type]; |
563 | qlast = ch->ch_last[type]; |
564 | if (!(flags & CHANNEL_SYNCF_ALLOC)) { |
565 | qfirst++; |
566 | } else if ((qlast - qfirst) > 1 && |
567 | !(flags & CHANNEL_SYNCF_ALLOC_BUF)) { |
568 | qlast--; |
569 | } |
570 | } |
571 | ASSERT(qfirst != qlast); |
572 | krings = NAKR(na, t: type); |
573 | |
574 | for (i = qfirst; i < qlast; i++) { |
575 | kring = krings + i; |
576 | s = kr_enter(kring, TRUE); |
577 | ASSERT(s == 0); |
578 | #if SK_LOG |
579 | if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) { |
580 | channel_sync_log1(0, "alloc-sync" , p, na, ch, |
581 | kring, i); |
582 | } |
583 | #endif /* SK_LOG */ |
584 | if (__improbable(kr_alloc_sync_prologue(kring, p) >= |
585 | kring->ckr_num_slots)) { |
586 | kr_log_bad_ring(kring); |
587 | if (!err) { |
588 | SK_DSC(p, |
589 | "EFAULT: kr_alloc_sync_prologue()" ); |
590 | err = EFAULT; |
591 | } |
592 | } else if (kring->ckr_na_sync(kring, p, |
593 | upp_sync_flags) == 0) { |
594 | kr_alloc_sync_finalize(kring, p); |
595 | } else { |
596 | if (!err) { |
597 | SK_DSC(p, |
598 | "EIO: ALLOC: ring->ckr_na_sync()" ); |
599 | err = EIO; |
600 | } |
601 | skywalk_kill_process(p, |
602 | SKYWALK_KILL_REASON_SYNC_FAILED | |
603 | SKYWALK_KILL_REASON_ALLOC_SYNC); |
604 | } |
605 | #if SK_LOG |
606 | if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) { |
607 | channel_sync_log2(0, "alloc-sync" , p, na, |
608 | kring, i); |
609 | } |
610 | #endif /* SK_LOG */ |
611 | kr_exit(kring); |
612 | } |
613 | } /* CHANNEL_SYNCF_ALLOC */ |
614 | |
615 | if (flags & CHANNEL_SYNCF_FREE) { |
616 | qfirst = ch->ch_first[NR_F]; |
617 | qlast = ch->ch_last[NR_F]; |
618 | ASSERT(qfirst != qlast); |
619 | krings = NAKR(na, t: NR_F); |
620 | |
621 | for (i = qfirst; i < qlast; i++) { |
622 | kring = krings + i; |
623 | s = kr_enter(kring, TRUE); |
624 | ASSERT(s == 0); |
625 | #if SK_LOG |
626 | if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) { |
627 | channel_sync_log1(0, "free-sync" , p, na, ch, |
628 | kring, i); |
629 | } |
630 | #endif /* SK_LOG */ |
631 | if (__improbable(kr_free_sync_prologue(kring, p) >= |
632 | kring->ckr_num_slots)) { |
633 | kr_log_bad_ring(kring); |
634 | if (!err) { |
635 | SK_DSC(p, |
636 | "EFAULT: kr_free_sync_prologue()" ); |
637 | err = EFAULT; |
638 | } |
639 | } else if (kring->ckr_na_sync(kring, p, |
640 | upp_sync_flags) == 0) { |
641 | kr_free_sync_finalize(kring, p); |
642 | } else { |
643 | if (!err) { |
644 | SK_DSC(p, |
645 | "EIO: FREE: ring->ckr_na_sync()" ); |
646 | err = EIO; |
647 | } |
648 | skywalk_kill_process(p, |
649 | SKYWALK_KILL_REASON_SYNC_FAILED | |
650 | SKYWALK_KILL_REASON_FREE_SYNC); |
651 | } |
652 | #if SK_LOG |
653 | if (__improbable((sk_verbose & SK_VERB_SYNC) != 0)) { |
654 | channel_sync_log2(0, "free-sync" , p, na, |
655 | kring, i); |
656 | } |
657 | #endif /* SK_LOG */ |
658 | kr_exit(kring); |
659 | } |
660 | } /* CHANNEL_SYNCF_FREE */ |
661 | |
662 | /* unmark thread with sync-in-progress flag */ |
663 | sk_sync_unprotect(protect); |
664 | |
665 | done: |
666 | lck_mtx_unlock(lck: &ch->ch_lock); |
667 | fp_drop(p, fd: uap->c, fp, locked: 0); |
668 | |
669 | VERIFY(!sk_is_sync_protected()); |
670 | |
671 | return err; |
672 | } |
673 | |
674 | int |
675 | __channel_get_opt(struct proc *p, struct __channel_get_opt_args *uap, |
676 | int *retval) |
677 | { |
678 | #pragma unused(retval) |
679 | struct fileproc *fp; |
680 | struct kern_channel *ch = NULL; |
681 | struct sockopt sopt; |
682 | uint32_t optlen; |
683 | int err = 0; |
684 | |
685 | AUDIT_ARG(fd, uap->c); |
686 | |
687 | err = fp_get_ftype(p, fd: uap->c, ftype: DTYPE_CHANNEL, ENODEV, fpp: &fp); |
688 | if (err != 0) { |
689 | SK_DSC(p, "fp_get_ftype err %u" , err); |
690 | return err; |
691 | } |
692 | ch = fp_get_data(fp); |
693 | |
694 | if (uap->aoptlen == USER_ADDR_NULL) { |
695 | SK_DSC(p, "EINVAL: uap->aoptlen == USER_ADDR_NULL" ); |
696 | err = EINVAL; |
697 | goto done; |
698 | } |
699 | |
700 | if (uap->aoptval != USER_ADDR_NULL) { |
701 | err = copyin(uap->aoptlen, &optlen, sizeof(optlen)); |
702 | if (err != 0) { |
703 | SK_DSC(p, "copyin err %u: aoptlen 0x%llx" , err, |
704 | SK_KVA(uap->aoptlen)); |
705 | goto done; |
706 | } |
707 | } else { |
708 | optlen = 0; |
709 | } |
710 | |
711 | bzero(s: &sopt, n: sizeof(sopt)); |
712 | sopt.sopt_dir = SOPT_GET; |
713 | sopt.sopt_name = uap->opt; |
714 | sopt.sopt_val = uap->aoptval; |
715 | sopt.sopt_valsize = optlen; |
716 | sopt.sopt_p = p; |
717 | |
718 | lck_mtx_lock(lck: &ch->ch_lock); |
719 | err = ch_get_opt(ch, &sopt); |
720 | lck_mtx_unlock(lck: &ch->ch_lock); |
721 | if (err == 0) { |
722 | optlen = (uint32_t)sopt.sopt_valsize; |
723 | err = copyout(&optlen, uap->aoptlen, sizeof(optlen)); |
724 | #if SK_LOG |
725 | if (err != 0) { |
726 | SK_DSC(p, "copyout err %u: aoptlen 0x%llx" , err, |
727 | SK_KVA(uap->aoptlen)); |
728 | } |
729 | #endif |
730 | } |
731 | |
732 | done: |
733 | fp_drop(p, fd: uap->c, fp, locked: 0); |
734 | |
735 | return err; |
736 | } |
737 | |
738 | int |
739 | __channel_set_opt(struct proc *p, struct __channel_set_opt_args *uap, |
740 | int *retval) |
741 | { |
742 | #pragma unused(retval) |
743 | struct fileproc *fp; |
744 | struct kern_channel *ch = NULL; |
745 | struct sockopt sopt; |
746 | int err = 0; |
747 | |
748 | AUDIT_ARG(fd, uap->c); |
749 | |
750 | err = fp_get_ftype(p, fd: uap->c, ftype: DTYPE_CHANNEL, ENODEV, fpp: &fp); |
751 | if (err != 0) { |
752 | SK_DSC(p, "fp_get_ftype err %u" , err); |
753 | return err; |
754 | } |
755 | ch = fp_get_data(fp); |
756 | |
757 | bzero(s: &sopt, n: sizeof(sopt)); |
758 | sopt.sopt_dir = SOPT_SET; |
759 | sopt.sopt_name = uap->opt; |
760 | sopt.sopt_val = uap->aoptval; |
761 | sopt.sopt_valsize = uap->optlen; |
762 | sopt.sopt_p = p; |
763 | |
764 | lck_mtx_lock(lck: &ch->ch_lock); |
765 | if (__improbable(ch->ch_flags & (CHANF_CLOSING | CHANF_DEFUNCT))) { |
766 | SK_DSC(p, "channel is closing/defunct" ); |
767 | err = ENXIO; |
768 | } else if (__improbable(ch->ch_na == NULL || |
769 | !NA_IS_ACTIVE(ch->ch_na) || na_reject_channel(ch, ch->ch_na))) { |
770 | SK_DSC(p, "channel is non-permissive" ); |
771 | err = ENXIO; |
772 | } else { |
773 | err = ch_set_opt(ch, &sopt); |
774 | } |
775 | lck_mtx_unlock(lck: &ch->ch_lock); |
776 | |
777 | fp_drop(p, fd: uap->c, fp, locked: 0); |
778 | |
779 | #if SK_LOG |
780 | if (err != 0) { |
781 | SK_DSC(p, "ch_set_opt() err %u" , err); |
782 | } |
783 | #endif |
784 | |
785 | return err; |
786 | } |
787 | |