1/*
2 * Copyright (c) 1998-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29/*
30 * Copyright (c) 1982, 1986, 1988, 1990, 1993
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)uipc_socket2.c 8.1 (Berkeley) 6/10/93
62 */
63/*
64 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
65 * support for mandatory and extensible security protections. This notice
66 * is included in support of clause 2.2 (b) of the Apple Public License,
67 * Version 2.0.
68 */
69
70#include <sys/param.h>
71#include <sys/systm.h>
72#include <sys/domain.h>
73#include <sys/kernel.h>
74#include <sys/proc_internal.h>
75#include <sys/kauth.h>
76#include <sys/malloc.h>
77#include <sys/mbuf.h>
78#include <sys/mcache.h>
79#include <sys/protosw.h>
80#include <sys/stat.h>
81#include <sys/socket.h>
82#include <sys/socketvar.h>
83#include <sys/signalvar.h>
84#include <sys/sysctl.h>
85#include <sys/syslog.h>
86#include <sys/unpcb.h>
87#include <sys/ev.h>
88#include <kern/locks.h>
89#include <net/route.h>
90#include <net/content_filter.h>
91#include <netinet/in.h>
92#include <netinet/in_pcb.h>
93#include <netinet/tcp_var.h>
94#include <sys/kdebug.h>
95#include <libkern/OSAtomic.h>
96
97#if CONFIG_MACF
98#include <security/mac_framework.h>
99#endif
100
101#include <mach/vm_param.h>
102
103#if MPTCP
104#include <netinet/mptcp_var.h>
105#endif
106
107#include <net/sockaddr_utils.h>
108
109extern uint32_t net_wake_pkt_debug;
110
111#define DBG_FNC_SBDROP NETDBG_CODE(DBG_NETSOCK, 4)
112#define DBG_FNC_SBAPPEND NETDBG_CODE(DBG_NETSOCK, 5)
113
114SYSCTL_DECL(_kern_ipc);
115
116__private_extern__ u_int32_t net_io_policy_throttle_best_effort = 0;
117SYSCTL_INT(_kern_ipc, OID_AUTO, throttle_best_effort,
118 CTLFLAG_RW | CTLFLAG_LOCKED, &net_io_policy_throttle_best_effort, 0, "");
119
120static inline void sbcompress(struct sockbuf *, struct mbuf *, struct mbuf *);
121static struct socket *sonewconn_internal(struct socket *, int);
122static int sbappendcontrol_internal(struct sockbuf *, struct mbuf *,
123 struct mbuf *);
124static void soevent_ifdenied(struct socket *);
125
126static int sbappendrecord_common(struct sockbuf *sb, struct mbuf *m0, boolean_t nodrop);
127static int sbappend_common(struct sockbuf *sb, struct mbuf *m, boolean_t nodrop);
128
129/*
130 * Primitive routines for operating on sockets and socket buffers
131 */
132static int soqlimitcompat = 1;
133static int soqlencomp = 0;
134
135/*
136 * Based on the number of mbuf clusters configured, high_sb_max and sb_max can
137 * get scaled up or down to suit that memory configuration. high_sb_max is a
138 * higher limit on sb_max that is checked when sb_max gets set through sysctl.
139 */
140uint32_t sb_max = SB_MAX;
141uint32_t high_sb_max = SB_MAX;
142
143static uint32_t sb_efficiency = 8; /* parameter for sbreserve() */
144
145uint32_t net_io_policy_log = 0; /* log socket policy changes */
146#if CONFIG_PROC_UUID_POLICY
147uint32_t net_io_policy_uuid = 1; /* enable UUID socket policy */
148#endif /* CONFIG_PROC_UUID_POLICY */
149
150/*
151 * Procedures to manipulate state flags of socket
152 * and do appropriate wakeups. Normal sequence from the
153 * active (originating) side is that soisconnecting() is
154 * called during processing of connect() call,
155 * resulting in an eventual call to soisconnected() if/when the
156 * connection is established. When the connection is torn down
157 * soisdisconnecting() is called during processing of disconnect() call,
158 * and soisdisconnected() is called when the connection to the peer
159 * is totally severed. The semantics of these routines are such that
160 * connectionless protocols can call soisconnected() and soisdisconnected()
161 * only, bypassing the in-progress calls when setting up a ``connection''
162 * takes no time.
163 *
164 * From the passive side, a socket is created with
165 * two queues of sockets: so_incomp for connections in progress
166 * and so_comp for connections already made and awaiting user acceptance.
167 * As a protocol is preparing incoming connections, it creates a socket
168 * structure queued on so_incomp by calling sonewconn(). When the connection
169 * is established, soisconnected() is called, and transfers the
170 * socket structure to so_comp, making it available to accept().
171 *
172 * If a socket is closed with sockets on either
173 * so_incomp or so_comp, these sockets are dropped.
174 *
175 * If higher level protocols are implemented in
176 * the kernel, the wakeups done here will sometimes
177 * cause software-interrupt process scheduling.
178 */
179void
180soisconnecting(struct socket *so)
181{
182 so->so_state &= ~(SS_ISCONNECTED | SS_ISDISCONNECTING);
183 so->so_state |= SS_ISCONNECTING;
184
185 sflt_notify(so, event: sock_evt_connecting, NULL);
186}
187
188void
189soisconnected(struct socket *so)
190{
191 /*
192 * If socket is subject to filter and is pending initial verdict,
193 * delay marking socket as connected and do not present the connected
194 * socket to user just yet.
195 */
196 if (cfil_sock_connected_pending_verdict(so)) {
197 return;
198 }
199
200 so->so_state &= ~(SS_ISCONNECTING | SS_ISDISCONNECTING | SS_ISCONFIRMING);
201 so->so_state |= SS_ISCONNECTED;
202
203 soreserve_preconnect(so, pre_cc: 0);
204
205 sflt_notify(so, event: sock_evt_connected, NULL);
206
207 if (so->so_head != NULL && (so->so_state & SS_INCOMP)) {
208 struct socket *head = so->so_head;
209 int locked = 0;
210
211 /*
212 * Enforce lock order when the protocol has per socket locks
213 */
214 if (head->so_proto->pr_getlock != NULL) {
215 socket_lock(so: head, refcount: 1);
216 so_acquire_accept_list(head, so);
217 locked = 1;
218 }
219 if (so->so_head == head && (so->so_state & SS_INCOMP)) {
220 so->so_state &= ~SS_INCOMP;
221 so->so_state |= SS_COMP;
222 TAILQ_REMOVE(&head->so_incomp, so, so_list);
223 TAILQ_INSERT_TAIL(&head->so_comp, so, so_list);
224 head->so_incqlen--;
225
226 /*
227 * We have to release the accept list in
228 * case a socket callback calls sock_accept()
229 */
230 if (locked != 0) {
231 so_release_accept_list(head);
232 socket_unlock(so, refcount: 0);
233 }
234 sorwakeup(so: head);
235 wakeup_one(chan: (caddr_t)&head->so_timeo);
236
237 if (locked != 0) {
238 socket_unlock(so: head, refcount: 1);
239 socket_lock(so, refcount: 0);
240 }
241 } else if (locked != 0) {
242 so_release_accept_list(head);
243 socket_unlock(so: head, refcount: 1);
244 }
245 } else {
246 wakeup(chan: (caddr_t)&so->so_timeo);
247 sorwakeup(so);
248 sowwakeup(so);
249 soevent(so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_CONNECTED |
250 SO_FILT_HINT_CONNINFO_UPDATED);
251 }
252}
253
254boolean_t
255socanwrite(struct socket *so)
256{
257 return (so->so_state & SS_ISCONNECTED) ||
258 !(so->so_proto->pr_flags & PR_CONNREQUIRED) ||
259 (so->so_flags1 & SOF1_PRECONNECT_DATA);
260}
261
262void
263soisdisconnecting(struct socket *so)
264{
265 so->so_state &= ~SS_ISCONNECTING;
266 so->so_state |= (SS_ISDISCONNECTING | SS_CANTRCVMORE | SS_CANTSENDMORE);
267 soevent(so, SO_FILT_HINT_LOCKED);
268 sflt_notify(so, event: sock_evt_disconnecting, NULL);
269 wakeup(chan: (caddr_t)&so->so_timeo);
270 sowwakeup(so);
271 sorwakeup(so);
272}
273
274void
275soisdisconnected(struct socket *so)
276{
277 so->so_state &= ~(SS_ISCONNECTING | SS_ISCONNECTED | SS_ISDISCONNECTING);
278 so->so_state |= (SS_CANTRCVMORE | SS_CANTSENDMORE | SS_ISDISCONNECTED);
279 soevent(so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_DISCONNECTED |
280 SO_FILT_HINT_CONNINFO_UPDATED);
281 sflt_notify(so, event: sock_evt_disconnected, NULL);
282 wakeup(chan: (caddr_t)&so->so_timeo);
283 sowwakeup(so);
284 sorwakeup(so);
285
286#if CONTENT_FILTER
287 /* Notify content filters as soon as we cannot send/receive data */
288 cfil_sock_notify_shutdown(so, SHUT_RDWR);
289#endif /* CONTENT_FILTER */
290}
291
292/*
293 * This function will issue a wakeup like soisdisconnected but it will not
294 * notify the socket filters. This will avoid unlocking the socket
295 * in the midst of closing it.
296 */
297void
298sodisconnectwakeup(struct socket *so)
299{
300 so->so_state &= ~(SS_ISCONNECTING | SS_ISCONNECTED | SS_ISDISCONNECTING);
301 so->so_state |= (SS_CANTRCVMORE | SS_CANTSENDMORE | SS_ISDISCONNECTED);
302 soevent(so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_DISCONNECTED |
303 SO_FILT_HINT_CONNINFO_UPDATED);
304 wakeup(chan: (caddr_t)&so->so_timeo);
305 sowwakeup(so);
306 sorwakeup(so);
307
308#if CONTENT_FILTER
309 /* Notify content filters as soon as we cannot send/receive data */
310 cfil_sock_notify_shutdown(so, SHUT_RDWR);
311#endif /* CONTENT_FILTER */
312}
313
314/*
315 * When an attempt at a new connection is noted on a socket
316 * which accepts connections, sonewconn is called. If the
317 * connection is possible (subject to space constraints, etc.)
318 * then we allocate a new structure, propoerly linked into the
319 * data structure of the original socket, and return this.
320 * Connstatus may be 0, or SO_ISCONFIRMING, or SO_ISCONNECTED.
321 */
322static struct socket *
323sonewconn_internal(struct socket *head, int connstatus)
324{
325 int so_qlen, error = 0;
326 struct socket *so;
327 lck_mtx_t *mutex_held;
328
329 if (head->so_proto->pr_getlock != NULL) {
330 mutex_held = (*head->so_proto->pr_getlock)(head, 0);
331 } else {
332 mutex_held = head->so_proto->pr_domain->dom_mtx;
333 }
334 LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
335
336 if (!soqlencomp) {
337 /*
338 * This is the default case; so_qlen represents the
339 * sum of both incomplete and completed queues.
340 */
341 so_qlen = head->so_qlen;
342 } else {
343 /*
344 * When kern.ipc.soqlencomp is set to 1, so_qlen
345 * represents only the completed queue. Since we
346 * cannot let the incomplete queue goes unbounded
347 * (in case of SYN flood), we cap the incomplete
348 * queue length to at most somaxconn, and use that
349 * as so_qlen so that we fail immediately below.
350 */
351 so_qlen = head->so_qlen - head->so_incqlen;
352 if (head->so_incqlen > somaxconn) {
353 so_qlen = somaxconn;
354 }
355 }
356
357 if (so_qlen >=
358 (soqlimitcompat ? head->so_qlimit : (3 * head->so_qlimit / 2))) {
359 return (struct socket *)0;
360 }
361 so = soalloc(waitok: 1, SOCK_DOM(head), type: head->so_type);
362 if (so == NULL) {
363 return (struct socket *)0;
364 }
365 /* check if head was closed during the soalloc */
366 if (head->so_proto == NULL) {
367 sodealloc(so);
368 return (struct socket *)0;
369 }
370
371 so->so_type = head->so_type;
372 so->so_family = head->so_family;
373 so->so_protocol = head->so_protocol;
374 so->so_options = head->so_options & ~SO_ACCEPTCONN;
375 so->so_linger = head->so_linger;
376 so->so_state = head->so_state | SS_NOFDREF;
377 so->so_proto = head->so_proto;
378 so->so_timeo = head->so_timeo;
379 so->so_pgid = head->so_pgid;
380 kauth_cred_ref(cred: head->so_cred);
381 so->so_cred = head->so_cred;
382 so->so_persona_id = head->so_persona_id;
383 so->last_pid = head->last_pid;
384 so->last_upid = head->last_upid;
385 memcpy(dst: so->last_uuid, src: head->last_uuid, n: sizeof(so->last_uuid));
386 if (head->so_flags & SOF_DELEGATED) {
387 so->e_pid = head->e_pid;
388 so->e_upid = head->e_upid;
389 memcpy(dst: so->e_uuid, src: head->e_uuid, n: sizeof(so->e_uuid));
390 }
391 /* inherit socket options stored in so_flags */
392 so->so_flags = head->so_flags &
393 (SOF_NOSIGPIPE | SOF_NOADDRAVAIL | SOF_REUSESHAREUID |
394 SOF_NOTIFYCONFLICT | SOF_BINDRANDOMPORT | SOF_NPX_SETOPTSHUT |
395 SOF_NODEFUNCT | SOF_PRIVILEGED_TRAFFIC_CLASS | SOF_NOTSENT_LOWAT |
396 SOF_DELEGATED);
397 so->so_flags1 |= SOF1_INBOUND;
398 so->so_usecount = 1;
399 so->next_lock_lr = 0;
400 so->next_unlock_lr = 0;
401
402 so->so_rcv.sb_flags |= SB_RECV; /* XXX */
403 so->so_rcv.sb_so = so->so_snd.sb_so = so;
404
405 /* inherit traffic management properties of listener */
406 so->so_flags1 |=
407 head->so_flags1 & (SOF1_TRAFFIC_MGT_SO_BACKGROUND | SOF1_TC_NET_SERV_TYPE |
408 SOF1_QOSMARKING_ALLOWED | SOF1_QOSMARKING_POLICY_OVERRIDE);
409 so->so_background_thread = head->so_background_thread;
410 so->so_traffic_class = head->so_traffic_class;
411 so->so_netsvctype = head->so_netsvctype;
412
413 if (soreserve(so, sndcc: head->so_snd.sb_hiwat, rcvcc: head->so_rcv.sb_hiwat)) {
414 sodealloc(so);
415 return (struct socket *)0;
416 }
417 so->so_rcv.sb_flags |= (head->so_rcv.sb_flags & SB_USRSIZE);
418 so->so_snd.sb_flags |= (head->so_snd.sb_flags & SB_USRSIZE);
419
420 /*
421 * Must be done with head unlocked to avoid deadlock
422 * for protocol with per socket mutexes.
423 */
424 if (head->so_proto->pr_unlock) {
425 socket_unlock(so: head, refcount: 0);
426 }
427 if (((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL) != 0) ||
428 error) {
429 sodealloc(so);
430 if (head->so_proto->pr_unlock) {
431 socket_lock(so: head, refcount: 0);
432 }
433 return (struct socket *)0;
434 }
435 if (head->so_proto->pr_unlock) {
436 socket_lock(so: head, refcount: 0);
437 /*
438 * Radar 7385998 Recheck that the head is still accepting
439 * to avoid race condition when head is getting closed.
440 */
441 if ((head->so_options & SO_ACCEPTCONN) == 0) {
442 so->so_state &= ~SS_NOFDREF;
443 soclose(so);
444 return (struct socket *)0;
445 }
446 }
447
448 if (so->so_proto->pr_copy_last_owner != NULL) {
449 (*so->so_proto->pr_copy_last_owner)(so, head);
450 }
451 os_atomic_inc(&so->so_proto->pr_domain->dom_refs, relaxed);
452
453 /* Insert in head appropriate lists */
454 so_acquire_accept_list(head, NULL);
455
456 so->so_head = head;
457
458 /*
459 * Since this socket is going to be inserted into the incomp
460 * queue, it can be picked up by another thread in
461 * tcp_dropdropablreq to get dropped before it is setup..
462 * To prevent this race, set in-progress flag which can be
463 * cleared later
464 */
465 so->so_flags |= SOF_INCOMP_INPROGRESS;
466
467 if (connstatus) {
468 TAILQ_INSERT_TAIL(&head->so_comp, so, so_list);
469 so->so_state |= SS_COMP;
470 } else {
471 TAILQ_INSERT_TAIL(&head->so_incomp, so, so_list);
472 so->so_state |= SS_INCOMP;
473 head->so_incqlen++;
474 }
475 head->so_qlen++;
476
477 so_release_accept_list(head);
478
479 /* Attach socket filters for this protocol */
480 sflt_initsock(so);
481
482 if (connstatus) {
483 so->so_state |= (short)connstatus;
484 sorwakeup(so: head);
485 wakeup(chan: (caddr_t)&head->so_timeo);
486 }
487 return so;
488}
489
490
491struct socket *
492sonewconn(struct socket *head, int connstatus, const struct sockaddr *from)
493{
494 int error = sflt_connectin(head, remote: from);
495 if (error) {
496 return NULL;
497 }
498
499 return sonewconn_internal(head, connstatus);
500}
501
502/*
503 * Socantsendmore indicates that no more data will be sent on the
504 * socket; it would normally be applied to a socket when the user
505 * informs the system that no more data is to be sent, by the protocol
506 * code (in case PRU_SHUTDOWN). Socantrcvmore indicates that no more data
507 * will be received, and will normally be applied to the socket by a
508 * protocol when it detects that the peer will send no more data.
509 * Data queued for reading in the socket may yet be read.
510 */
511
512void
513socantsendmore(struct socket *so)
514{
515 so->so_state |= SS_CANTSENDMORE;
516 soevent(so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_CANTSENDMORE);
517 sflt_notify(so, event: sock_evt_cantsendmore, NULL);
518 sowwakeup(so);
519}
520
521void
522socantrcvmore(struct socket *so)
523{
524 so->so_state |= SS_CANTRCVMORE;
525 soevent(so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_CANTRCVMORE);
526 sflt_notify(so, event: sock_evt_cantrecvmore, NULL);
527 sorwakeup(so);
528}
529
530/*
531 * Wait for data to arrive at/drain from a socket buffer.
532 */
533int
534sbwait(struct sockbuf *sb)
535{
536 boolean_t nointr = (sb->sb_flags & SB_NOINTR);
537 void *lr_saved = __builtin_return_address(0);
538 struct socket *so = sb->sb_so;
539 lck_mtx_t *mutex_held;
540 struct timespec ts;
541 int error = 0;
542
543 if (so == NULL) {
544 panic("%s: null so, sb=%p sb_flags=0x%x lr=%p",
545 __func__, sb, sb->sb_flags, lr_saved);
546 /* NOTREACHED */
547 } else if (so->so_usecount < 1) {
548 panic("%s: sb=%p sb_flags=0x%x sb_so=%p usecount=%d lr=%p "
549 "lrh= %s\n", __func__, sb, sb->sb_flags, so,
550 so->so_usecount, lr_saved, solockhistory_nr(so));
551 /* NOTREACHED */
552 }
553
554 if ((so->so_state & SS_DRAINING) || (so->so_flags & SOF_DEFUNCT)) {
555 error = EBADF;
556 if (so->so_flags & SOF_DEFUNCT) {
557 SODEFUNCTLOG("%s[%d, %s]: defunct so 0x%llu [%d,%d] "
558 "(%d)\n", __func__, proc_selfpid(),
559 proc_best_name(current_proc()),
560 so->so_gencnt,
561 SOCK_DOM(so), SOCK_TYPE(so), error);
562 }
563 return error;
564 }
565
566 if (so->so_proto->pr_getlock != NULL) {
567 mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK);
568 } else {
569 mutex_held = so->so_proto->pr_domain->dom_mtx;
570 }
571
572 LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
573
574 ts.tv_sec = sb->sb_timeo.tv_sec;
575 ts.tv_nsec = sb->sb_timeo.tv_usec * 1000;
576
577 sb->sb_waiters++;
578 VERIFY(sb->sb_waiters != 0);
579
580 error = msleep(chan: (caddr_t)&sb->sb_cc, mtx: mutex_held,
581 pri: nointr ? PSOCK : PSOCK | PCATCH,
582 wmesg: nointr ? "sbwait_nointr" : "sbwait", ts: &ts);
583
584 VERIFY(sb->sb_waiters != 0);
585 sb->sb_waiters--;
586
587 if (so->so_usecount < 1) {
588 panic("%s: 2 sb=%p sb_flags=0x%x sb_so=%p usecount=%d lr=%p "
589 "lrh= %s\n", __func__, sb, sb->sb_flags, so,
590 so->so_usecount, lr_saved, solockhistory_nr(so));
591 /* NOTREACHED */
592 }
593
594 if ((so->so_state & SS_DRAINING) || (so->so_flags & SOF_DEFUNCT)) {
595 error = EBADF;
596 if (so->so_flags & SOF_DEFUNCT) {
597 SODEFUNCTLOG("%s[%d, %s]: defunct so 0x%llu [%d,%d] "
598 "(%d)\n", __func__, proc_selfpid(),
599 proc_best_name(current_proc()),
600 so->so_gencnt,
601 SOCK_DOM(so), SOCK_TYPE(so), error);
602 }
603 }
604
605 return error;
606}
607
608void
609sbwakeup(struct sockbuf *sb)
610{
611 if (sb->sb_waiters > 0) {
612 wakeup(chan: (caddr_t)&sb->sb_cc);
613 }
614}
615
616/*
617 * Wakeup processes waiting on a socket buffer.
618 * Do asynchronous notification via SIGIO
619 * if the socket has the SS_ASYNC flag set.
620 */
621void
622sowakeup(struct socket *so, struct sockbuf *sb, struct socket *so2)
623{
624 if (so->so_flags & SOF_DEFUNCT) {
625 SODEFUNCTLOG("%s[%d, %s]: defunct so 0x%llu [%d,%d] si 0x%x, "
626 "fl 0x%x [%s]\n", __func__, proc_selfpid(),
627 proc_best_name(current_proc()),
628 so->so_gencnt, SOCK_DOM(so),
629 SOCK_TYPE(so), (uint32_t)sb->sb_sel.si_flags, sb->sb_flags,
630 (sb->sb_flags & SB_RECV) ? "rcv" : "snd");
631 }
632
633 sb->sb_flags &= ~SB_SEL;
634 selwakeup(&sb->sb_sel);
635 sbwakeup(sb);
636 if (so->so_state & SS_ASYNC) {
637 if (so->so_pgid < 0) {
638 gsignal(pgid: -so->so_pgid, SIGIO);
639 } else if (so->so_pgid > 0) {
640 proc_signal(pid: so->so_pgid, SIGIO);
641 }
642 }
643 if (sb->sb_flags & SB_KNOTE) {
644 KNOTE(&sb->sb_sel.si_note, SO_FILT_HINT_LOCKED);
645 }
646 if (sb->sb_flags & SB_UPCALL) {
647 void (*sb_upcall)(struct socket *, void *, int);
648 caddr_t sb_upcallarg;
649 int lock = !(sb->sb_flags & SB_UPCALL_LOCK);
650
651 sb_upcall = sb->sb_upcall;
652 sb_upcallarg = sb->sb_upcallarg;
653 /* Let close know that we're about to do an upcall */
654 so->so_upcallusecount++;
655
656 if (lock) {
657 if (so2) {
658 struct unpcb *unp = sotounpcb(so2);
659 unp->unp_flags |= UNP_DONTDISCONNECT;
660 unp->rw_thrcount++;
661
662 socket_unlock(so: so2, refcount: 0);
663 }
664 socket_unlock(so, refcount: 0);
665 }
666 (*sb_upcall)(so, sb_upcallarg, M_DONTWAIT);
667 if (lock) {
668 if (so2 && so > so2) {
669 struct unpcb *unp;
670 socket_lock(so: so2, refcount: 0);
671
672 unp = sotounpcb(so2);
673 unp->rw_thrcount--;
674 if (unp->rw_thrcount == 0) {
675 unp->unp_flags &= ~UNP_DONTDISCONNECT;
676 wakeup(chan: unp);
677 }
678 }
679
680 socket_lock(so, refcount: 0);
681
682 if (so2 && so < so2) {
683 struct unpcb *unp;
684 socket_lock(so: so2, refcount: 0);
685
686 unp = sotounpcb(so2);
687 unp->rw_thrcount--;
688 if (unp->rw_thrcount == 0) {
689 unp->unp_flags &= ~UNP_DONTDISCONNECT;
690 wakeup(chan: unp);
691 }
692 }
693 }
694
695 so->so_upcallusecount--;
696 /* Tell close that it's safe to proceed */
697 if ((so->so_flags & SOF_CLOSEWAIT) &&
698 so->so_upcallusecount == 0) {
699 wakeup(chan: (caddr_t)&so->so_upcallusecount);
700 }
701 }
702#if CONTENT_FILTER
703 /*
704 * Trap disconnection events for content filters
705 */
706 if ((so->so_flags & SOF_CONTENT_FILTER) != 0) {
707 if ((sb->sb_flags & SB_RECV)) {
708 if (so->so_state & (SS_CANTRCVMORE)) {
709 cfil_sock_notify_shutdown(so, SHUT_RD);
710 }
711 } else {
712 if (so->so_state & (SS_CANTSENDMORE)) {
713 cfil_sock_notify_shutdown(so, SHUT_WR);
714 }
715 }
716 }
717#endif /* CONTENT_FILTER */
718}
719
720/*
721 * Socket buffer (struct sockbuf) utility routines.
722 *
723 * Each socket contains two socket buffers: one for sending data and
724 * one for receiving data. Each buffer contains a queue of mbufs,
725 * information about the number of mbufs and amount of data in the
726 * queue, and other fields allowing select() statements and notification
727 * on data availability to be implemented.
728 *
729 * Data stored in a socket buffer is maintained as a list of records.
730 * Each record is a list of mbufs chained together with the m_next
731 * field. Records are chained together with the m_nextpkt field. The upper
732 * level routine soreceive() expects the following conventions to be
733 * observed when placing information in the receive buffer:
734 *
735 * 1. If the protocol requires each message be preceded by the sender's
736 * name, then a record containing that name must be present before
737 * any associated data (mbuf's must be of type MT_SONAME).
738 * 2. If the protocol supports the exchange of ``access rights'' (really
739 * just additional data associated with the message), and there are
740 * ``rights'' to be received, then a record containing this data
741 * should be present (mbuf's must be of type MT_RIGHTS).
742 * 3. If a name or rights record exists, then it must be followed by
743 * a data record, perhaps of zero length.
744 *
745 * Before using a new socket structure it is first necessary to reserve
746 * buffer space to the socket, by calling sbreserve(). This should commit
747 * some of the available buffer space in the system buffer pool for the
748 * socket (currently, it does nothing but enforce limits). The space
749 * should be released by calling sbrelease() when the socket is destroyed.
750 */
751
752/*
753 * Returns: 0 Success
754 * ENOBUFS
755 */
756int
757soreserve(struct socket *so, uint32_t sndcc, uint32_t rcvcc)
758{
759 if (sbreserve(sb: &so->so_snd, cc: sndcc) == 0) {
760 goto bad;
761 } else {
762 so->so_snd.sb_idealsize = sndcc;
763 }
764
765 if (sbreserve(sb: &so->so_rcv, cc: rcvcc) == 0) {
766 goto bad2;
767 } else {
768 so->so_rcv.sb_idealsize = rcvcc;
769 }
770
771 if (so->so_rcv.sb_lowat == 0) {
772 so->so_rcv.sb_lowat = 1;
773 }
774 if (so->so_snd.sb_lowat == 0) {
775 so->so_snd.sb_lowat = MCLBYTES;
776 }
777 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) {
778 so->so_snd.sb_lowat = so->so_snd.sb_hiwat;
779 }
780 return 0;
781bad2:
782 so->so_snd.sb_flags &= ~SB_SEL;
783 selthreadclear(&so->so_snd.sb_sel);
784 sbrelease(sb: &so->so_snd);
785bad:
786 return ENOBUFS;
787}
788
789void
790soreserve_preconnect(struct socket *so, unsigned int pre_cc)
791{
792 /* As of now, same bytes for both preconnect read and write */
793 so->so_snd.sb_preconn_hiwat = pre_cc;
794 so->so_rcv.sb_preconn_hiwat = pre_cc;
795}
796
797/*
798 * Allot mbufs to a sockbuf.
799 * Attempt to scale mbmax so that mbcnt doesn't become limiting
800 * if buffering efficiency is near the normal case.
801 */
802int
803sbreserve(struct sockbuf *sb, uint32_t cc)
804{
805 if (cc > sb_max) {
806 /* We would not end up changing sb_cc, so return 0 */
807 if (sb->sb_hiwat == sb_max) {
808 return 0;
809 }
810 cc = sb_max;
811 }
812 if (cc > sb->sb_hiwat && (sb->sb_flags & SB_LIMITED)) {
813 return 0;
814 }
815 sb->sb_hiwat = cc;
816 sb->sb_mbmax = cc * sb_efficiency;
817 if (sb->sb_lowat > sb->sb_hiwat) {
818 sb->sb_lowat = sb->sb_hiwat;
819 }
820 return 1;
821}
822
823/*
824 * Free mbufs held by a socket, and reserved mbuf space.
825 */
826/* WARNING needs to do selthreadclear() before calling this */
827void
828sbrelease(struct sockbuf *sb)
829{
830 sbflush(sb);
831 sb->sb_hiwat = 0;
832 sb->sb_mbmax = 0;
833}
834
835/*
836 * Routines to add and remove
837 * data from an mbuf queue.
838 *
839 * The routines sbappend() or sbappendrecord() are normally called to
840 * append new mbufs to a socket buffer, after checking that adequate
841 * space is available, comparing the function sbspace() with the amount
842 * of data to be added. sbappendrecord() differs from sbappend() in
843 * that data supplied is treated as the beginning of a new record.
844 * To place a sender's address, optional access rights, and data in a
845 * socket receive buffer, sbappendaddr() should be used. To place
846 * access rights and data in a socket receive buffer, sbappendrights()
847 * should be used. In either case, the new data begins a new record.
848 * Note that unlike sbappend() and sbappendrecord(), these routines check
849 * for the caller that there will be enough space to store the data.
850 * Each fails if there is not enough space, or if it cannot find mbufs
851 * to store additional information in.
852 *
853 * Reliable protocols may use the socket send buffer to hold data
854 * awaiting acknowledgement. Data is normally copied from a socket
855 * send buffer in a protocol with m_copy for output to a peer,
856 * and then removing the data from the socket buffer with sbdrop()
857 * or sbdroprecord() when the data is acknowledged by the peer.
858 */
859
860/*
861 * Append mbuf chain m to the last record in the
862 * socket buffer sb. The additional space associated
863 * the mbuf chain is recorded in sb. Empty mbufs are
864 * discarded and mbufs are compacted where possible.
865 */
866static int
867sbappend_common(struct sockbuf *sb, struct mbuf *m, boolean_t nodrop)
868{
869 struct socket *so = sb->sb_so;
870 struct soflow_hash_entry *dgram_flow_entry = NULL;
871
872 if (m == NULL || (sb->sb_flags & SB_DROP)) {
873 if (m != NULL && !nodrop) {
874 m_freem(m);
875 }
876 return 0;
877 }
878
879 SBLASTRECORDCHK(sb, "sbappend 1");
880
881 if (sb->sb_lastrecord != NULL && (sb->sb_mbtail->m_flags & M_EOR)) {
882 return sbappendrecord_common(sb, m0: m, nodrop);
883 }
884
885 if (SOCK_DOM(sb->sb_so) == PF_INET || SOCK_DOM(sb->sb_so) == PF_INET6) {
886 ASSERT(nodrop == FALSE);
887
888 if (NEED_DGRAM_FLOW_TRACKING(so)) {
889 dgram_flow_entry = soflow_get_flow(so, NULL, NULL, NULL, m != NULL ? m_length(m) : 0, false, (m != NULL && m->m_pkthdr.rcvif) ? m->m_pkthdr.rcvif->if_index : 0);
890 }
891
892 if (sb->sb_flags & SB_RECV && !(m && m->m_flags & M_SKIPCFIL)) {
893 int error = sflt_data_in(so, NULL, data: &m, NULL, flags: 0);
894 SBLASTRECORDCHK(sb, "sbappend 2");
895
896#if CONTENT_FILTER
897 if (error == 0) {
898 error = cfil_sock_data_in(so, NULL, data: m, NULL, flags: 0, dgram_flow_entry);
899 }
900#endif /* CONTENT_FILTER */
901
902 if (error != 0) {
903 if (error != EJUSTRETURN) {
904 m_freem(m);
905 }
906 if (dgram_flow_entry != NULL) {
907 soflow_free_flow(dgram_flow_entry);
908 }
909 return 0;
910 }
911 } else if (m) {
912 m->m_flags &= ~M_SKIPCFIL;
913 }
914
915 if (dgram_flow_entry != NULL) {
916 soflow_free_flow(dgram_flow_entry);
917 }
918 }
919
920 /* If this is the first record, it's also the last record */
921 if (sb->sb_lastrecord == NULL) {
922 sb->sb_lastrecord = m;
923 }
924
925 sbcompress(sb, m, sb->sb_mbtail);
926 SBLASTRECORDCHK(sb, "sbappend 3");
927 return 1;
928}
929
930int
931sbappend(struct sockbuf *sb, struct mbuf *m)
932{
933 return sbappend_common(sb, m, FALSE);
934}
935
936int
937sbappend_nodrop(struct sockbuf *sb, struct mbuf *m)
938{
939 return sbappend_common(sb, m, TRUE);
940}
941
942/*
943 * Similar to sbappend, except that this is optimized for stream sockets.
944 */
945int
946sbappendstream(struct sockbuf *sb, struct mbuf *m)
947{
948 struct soflow_hash_entry *dgram_flow_entry = NULL;
949 struct socket *so = sb->sb_so;
950
951 if (m == NULL || (sb->sb_flags & SB_DROP)) {
952 if (m != NULL) {
953 m_freem(m);
954 }
955 return 0;
956 }
957
958 if (m->m_nextpkt != NULL || (sb->sb_mb != sb->sb_lastrecord)) {
959 panic("sbappendstream: nexpkt %p || mb %p != lastrecord %p",
960 m->m_nextpkt, sb->sb_mb, sb->sb_lastrecord);
961 /* NOTREACHED */
962 }
963
964 SBLASTMBUFCHK(sb, __func__);
965
966 if (SOCK_DOM(sb->sb_so) == PF_INET || SOCK_DOM(sb->sb_so) == PF_INET6) {
967 if (NEED_DGRAM_FLOW_TRACKING(so)) {
968 dgram_flow_entry = soflow_get_flow(so, NULL, NULL, NULL, m != NULL ? m_length(m) : 0, false, (m != NULL && m->m_pkthdr.rcvif) ? m->m_pkthdr.rcvif->if_index : 0);
969 }
970
971 if (sb->sb_flags & SB_RECV && !(m && m->m_flags & M_SKIPCFIL)) {
972 int error = sflt_data_in(so, NULL, data: &m, NULL, flags: 0);
973 SBLASTRECORDCHK(sb, "sbappendstream 1");
974
975#if CONTENT_FILTER
976 if (error == 0) {
977 error = cfil_sock_data_in(so, NULL, data: m, NULL, flags: 0, dgram_flow_entry);
978 }
979#endif /* CONTENT_FILTER */
980
981 if (error != 0) {
982 if (error != EJUSTRETURN) {
983 m_freem(m);
984 }
985 if (dgram_flow_entry != NULL) {
986 soflow_free_flow(dgram_flow_entry);
987 }
988 return 0;
989 }
990 } else if (m) {
991 m->m_flags &= ~M_SKIPCFIL;
992 }
993
994 if (dgram_flow_entry != NULL) {
995 soflow_free_flow(dgram_flow_entry);
996 }
997 }
998
999 sbcompress(sb, m, sb->sb_mbtail);
1000 sb->sb_lastrecord = sb->sb_mb;
1001 SBLASTRECORDCHK(sb, "sbappendstream 2");
1002 return 1;
1003}
1004
1005#ifdef SOCKBUF_DEBUG
1006void
1007sbcheck(struct sockbuf *sb)
1008{
1009 struct mbuf *m;
1010 struct mbuf *n = 0;
1011 u_int32_t len = 0, mbcnt = 0;
1012 lck_mtx_t *mutex_held;
1013
1014 if (sb->sb_so->so_proto->pr_getlock != NULL) {
1015 mutex_held = (*sb->sb_so->so_proto->pr_getlock)(sb->sb_so, 0);
1016 } else {
1017 mutex_held = sb->sb_so->so_proto->pr_domain->dom_mtx;
1018 }
1019
1020 LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
1021
1022 if (sbchecking == 0) {
1023 return;
1024 }
1025
1026 for (m = sb->sb_mb; m; m = n) {
1027 n = m->m_nextpkt;
1028 for (; m; m = m->m_next) {
1029 len += m->m_len;
1030 mbcnt += _MSIZE;
1031 /* XXX pretty sure this is bogus */
1032 if (m->m_flags & M_EXT) {
1033 mbcnt += m->m_ext.ext_size;
1034 }
1035 }
1036 }
1037 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) {
1038 panic("cc %ld != %ld || mbcnt %ld != %ld", len, sb->sb_cc,
1039 mbcnt, sb->sb_mbcnt);
1040 }
1041}
1042#endif
1043
1044void
1045sblastrecordchk(struct sockbuf *sb, const char *where)
1046{
1047 struct mbuf *m = sb->sb_mb;
1048
1049 while (m && m->m_nextpkt) {
1050 m = m->m_nextpkt;
1051 }
1052
1053 if (m != sb->sb_lastrecord) {
1054 printf("sblastrecordchk: mb 0x%llx lastrecord 0x%llx "
1055 "last 0x%llx\n",
1056 (uint64_t)VM_KERNEL_ADDRPERM(sb->sb_mb),
1057 (uint64_t)VM_KERNEL_ADDRPERM(sb->sb_lastrecord),
1058 (uint64_t)VM_KERNEL_ADDRPERM(m));
1059 printf("packet chain:\n");
1060 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) {
1061 printf("\t0x%llx\n", (uint64_t)VM_KERNEL_ADDRPERM(m));
1062 }
1063 panic("sblastrecordchk from %s", where);
1064 }
1065}
1066
1067void
1068sblastmbufchk(struct sockbuf *sb, const char *where)
1069{
1070 struct mbuf *m = sb->sb_mb;
1071 struct mbuf *n;
1072
1073 while (m && m->m_nextpkt) {
1074 m = m->m_nextpkt;
1075 }
1076
1077 while (m && m->m_next) {
1078 m = m->m_next;
1079 }
1080
1081 if (m != sb->sb_mbtail) {
1082 printf("sblastmbufchk: mb 0x%llx mbtail 0x%llx last 0x%llx\n",
1083 (uint64_t)VM_KERNEL_ADDRPERM(sb->sb_mb),
1084 (uint64_t)VM_KERNEL_ADDRPERM(sb->sb_mbtail),
1085 (uint64_t)VM_KERNEL_ADDRPERM(m));
1086 printf("packet tree:\n");
1087 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) {
1088 printf("\t");
1089 for (n = m; n != NULL; n = n->m_next) {
1090 printf("0x%llx ",
1091 (uint64_t)VM_KERNEL_ADDRPERM(n));
1092 }
1093 printf("\n");
1094 }
1095 panic("sblastmbufchk from %s", where);
1096 }
1097}
1098
1099/*
1100 * Similar to sbappend, except the mbuf chain begins a new record.
1101 */
1102static int
1103sbappendrecord_common(struct sockbuf *sb, struct mbuf *m0, boolean_t nodrop)
1104{
1105 struct soflow_hash_entry *dgram_flow_entry = NULL;
1106 struct socket *so = sb->sb_so;
1107 struct mbuf *m;
1108 int space = 0;
1109
1110 if (m0 == NULL || (sb->sb_flags & SB_DROP)) {
1111 if (m0 != NULL && nodrop == FALSE) {
1112 m_freem(m0);
1113 }
1114 return 0;
1115 }
1116
1117 for (m = m0; m != NULL; m = m->m_next) {
1118 space += m->m_len;
1119 }
1120
1121 if (space > sbspace(sb) && !(sb->sb_flags & SB_UNIX)) {
1122 if (nodrop == FALSE) {
1123 m_freem(m0);
1124 }
1125 return 0;
1126 }
1127
1128 if (SOCK_DOM(sb->sb_so) == PF_INET || SOCK_DOM(sb->sb_so) == PF_INET6) {
1129 ASSERT(nodrop == FALSE);
1130
1131 if (NEED_DGRAM_FLOW_TRACKING(so)) {
1132 dgram_flow_entry = soflow_get_flow(so, NULL, NULL, NULL, m0 != NULL ? m_length(m0) : 0, false, (m0 != NULL && m0->m_pkthdr.rcvif) ? m0->m_pkthdr.rcvif->if_index : 0);
1133 }
1134
1135 if (sb->sb_flags & SB_RECV && !(m0 && m0->m_flags & M_SKIPCFIL)) {
1136 int error = sflt_data_in(so: sb->sb_so, NULL, data: &m0, NULL,
1137 flags: sock_data_filt_flag_record);
1138
1139#if CONTENT_FILTER
1140 if (error == 0) {
1141 error = cfil_sock_data_in(so: sb->sb_so, NULL, data: m0, NULL, flags: 0, dgram_flow_entry);
1142 }
1143#endif /* CONTENT_FILTER */
1144
1145 if (error != 0) {
1146 SBLASTRECORDCHK(sb, "sbappendrecord 1");
1147 if (error != EJUSTRETURN) {
1148 m_freem(m0);
1149 }
1150 if (dgram_flow_entry != NULL) {
1151 soflow_free_flow(dgram_flow_entry);
1152 }
1153 return 0;
1154 }
1155 } else if (m0) {
1156 m0->m_flags &= ~M_SKIPCFIL;
1157 }
1158
1159 if (dgram_flow_entry != NULL) {
1160 soflow_free_flow(dgram_flow_entry);
1161 }
1162 }
1163
1164 /*
1165 * Note this permits zero length records.
1166 */
1167 sballoc(sb, m: m0);
1168 SBLASTRECORDCHK(sb, "sbappendrecord 2");
1169 if (sb->sb_lastrecord != NULL) {
1170 sb->sb_lastrecord->m_nextpkt = m0;
1171 } else {
1172 sb->sb_mb = m0;
1173 }
1174 sb->sb_lastrecord = m0;
1175 sb->sb_mbtail = m0;
1176
1177 m = m0->m_next;
1178 m0->m_next = 0;
1179 if (m && (m0->m_flags & M_EOR)) {
1180 m0->m_flags &= ~M_EOR;
1181 m->m_flags |= M_EOR;
1182 }
1183 sbcompress(sb, m, m0);
1184 SBLASTRECORDCHK(sb, "sbappendrecord 3");
1185 return 1;
1186}
1187
1188int
1189sbappendrecord(struct sockbuf *sb, struct mbuf *m0)
1190{
1191 return sbappendrecord_common(sb, m0, FALSE);
1192}
1193
1194int
1195sbappendrecord_nodrop(struct sockbuf *sb, struct mbuf *m0)
1196{
1197 return sbappendrecord_common(sb, m0, TRUE);
1198}
1199
1200/*
1201 * Concatenate address (optional), control (optional) and data into one
1202 * single mbuf chain. If sockbuf *sb is passed in, space check will be
1203 * performed.
1204 *
1205 * Returns: mbuf chain pointer if succeeded, NULL if failed
1206 */
1207struct mbuf *
1208sbconcat_mbufs(struct sockbuf *sb, struct sockaddr *asa, struct mbuf *m0, struct mbuf *control)
1209{
1210 struct mbuf *m = NULL, *n = NULL;
1211 int space = 0;
1212
1213 if (m0 && (m0->m_flags & M_PKTHDR) == 0) {
1214 panic("sbconcat_mbufs");
1215 }
1216
1217 if (m0) {
1218 space += m0->m_pkthdr.len;
1219 }
1220 for (n = control; n; n = n->m_next) {
1221 space += n->m_len;
1222 if (n->m_next == 0) { /* keep pointer to last control buf */
1223 break;
1224 }
1225 }
1226
1227 if (asa != NULL) {
1228 _CASSERT(sizeof(asa->sa_len) == sizeof(__uint8_t));
1229 if (MLEN <= UINT8_MAX && asa->sa_len > MLEN) {
1230 return NULL;
1231 }
1232 space += asa->sa_len;
1233 }
1234
1235 if (sb != NULL && space > sbspace(sb)) {
1236 return NULL;
1237 }
1238
1239 if (n) {
1240 n->m_next = m0; /* concatenate data to control */
1241 } else {
1242 control = m0;
1243 }
1244
1245 if (asa != NULL) {
1246 MGET(m, M_DONTWAIT, MT_SONAME);
1247 if (m == 0) {
1248 if (n) {
1249 /* unchain control and data if necessary */
1250 n->m_next = NULL;
1251 }
1252 return NULL;
1253 }
1254 m->m_len = asa->sa_len;
1255 bcopy(src: (caddr_t)asa, mtod(m, caddr_t), n: asa->sa_len);
1256
1257 m->m_next = control;
1258 } else {
1259 m = control;
1260 }
1261
1262 return m;
1263}
1264
1265/*
1266 * Queue mbuf chain to the receive queue of a socket.
1267 * Parameter space is the total len of the mbuf chain.
1268 * If passed in, sockbuf space will be checked.
1269 *
1270 * Returns: 0 Invalid mbuf chain
1271 * 1 Success
1272 */
1273int
1274sbappendchain(struct sockbuf *sb, struct mbuf *m, int space)
1275{
1276 struct mbuf *n, *nlast;
1277
1278 if (m == NULL) {
1279 return 0;
1280 }
1281
1282 if (space != 0 && space > sbspace(sb)) {
1283 return 0;
1284 }
1285
1286 for (n = m; n->m_next != NULL; n = n->m_next) {
1287 sballoc(sb, m: n);
1288 }
1289 sballoc(sb, m: n);
1290 nlast = n;
1291
1292 if (sb->sb_lastrecord != NULL) {
1293 sb->sb_lastrecord->m_nextpkt = m;
1294 } else {
1295 sb->sb_mb = m;
1296 }
1297 sb->sb_lastrecord = m;
1298 sb->sb_mbtail = nlast;
1299
1300 SBLASTMBUFCHK(sb, __func__);
1301 SBLASTRECORDCHK(sb, "sbappendadddr 2");
1302 return 1;
1303}
1304
1305/*
1306 * Returns: 0 Error: No space/out of mbufs/etc.
1307 * 1 Success
1308 *
1309 * Imputed: (*error_out) errno for error
1310 * ENOBUFS
1311 * sflt_data_in:??? [whatever a filter author chooses]
1312 */
1313int
1314sbappendaddr(struct sockbuf *sb, struct sockaddr *asa, struct mbuf *m0,
1315 struct mbuf *control, int *error_out)
1316{
1317 int result = 0;
1318 boolean_t sb_unix = (sb->sb_flags & SB_UNIX);
1319 struct mbuf *mbuf_chain = NULL;
1320 struct soflow_hash_entry *dgram_flow_entry = NULL;
1321 struct socket *so = sb->sb_so;
1322
1323 if (error_out) {
1324 *error_out = 0;
1325 }
1326
1327 if (m0 && (m0->m_flags & M_PKTHDR) == 0) {
1328 panic("sbappendaddrorfree");
1329 }
1330
1331 if (sb->sb_flags & SB_DROP) {
1332 if (m0 != NULL) {
1333 m_freem(m0);
1334 }
1335 if (control != NULL && !sb_unix) {
1336 m_freem(control);
1337 }
1338 if (error_out != NULL) {
1339 *error_out = EINVAL;
1340 }
1341 return 0;
1342 }
1343
1344 if (SOCK_DOM(sb->sb_so) == PF_INET || SOCK_DOM(sb->sb_so) == PF_INET6) {
1345 /* Call socket data in filters */
1346
1347 if (NEED_DGRAM_FLOW_TRACKING(so)) {
1348 dgram_flow_entry = soflow_get_flow(so, NULL, asa, control, m0 != NULL ? m_length(m0) : 0, false, (m0 != NULL && m0->m_pkthdr.rcvif) ? m0->m_pkthdr.rcvif->if_index : 0);
1349 }
1350
1351 if (sb->sb_flags & SB_RECV && !(m0 && m0->m_flags & M_SKIPCFIL)) {
1352 int error;
1353 error = sflt_data_in(so: sb->sb_so, from: asa, data: &m0, control: &control, flags: 0);
1354 SBLASTRECORDCHK(sb, __func__);
1355
1356#if CONTENT_FILTER
1357 if (error == 0) {
1358 error = cfil_sock_data_in(so: sb->sb_so, from: asa, data: m0, control,
1359 flags: 0, dgram_flow_entry);
1360 }
1361#endif /* CONTENT_FILTER */
1362
1363 if (error) {
1364 if (error != EJUSTRETURN) {
1365 if (m0) {
1366 m_freem(m0);
1367 }
1368 if (control != NULL && !sb_unix) {
1369 m_freem(control);
1370 }
1371 if (error_out) {
1372 *error_out = error;
1373 }
1374 }
1375 if (dgram_flow_entry != NULL) {
1376 soflow_free_flow(dgram_flow_entry);
1377 }
1378 return 0;
1379 }
1380 } else if (m0) {
1381 m0->m_flags &= ~M_SKIPCFIL;
1382 }
1383
1384 if (dgram_flow_entry != NULL) {
1385 soflow_free_flow(dgram_flow_entry);
1386 }
1387 }
1388
1389 mbuf_chain = sbconcat_mbufs(sb, asa, m0, control);
1390 SBLASTRECORDCHK(sb, "sbappendadddr 1");
1391 result = sbappendchain(sb, m: mbuf_chain, space: 0);
1392 if (result == 0) {
1393 if (m0) {
1394 m_freem(m0);
1395 }
1396 if (control != NULL && !sb_unix) {
1397 m_freem(control);
1398 }
1399 if (error_out) {
1400 *error_out = ENOBUFS;
1401 }
1402 }
1403
1404 return result;
1405}
1406
1407inline boolean_t
1408is_cmsg_valid(struct mbuf *control, struct cmsghdr *cmsg)
1409{
1410 if (cmsg == NULL) {
1411 return FALSE;
1412 }
1413
1414 if (cmsg->cmsg_len < sizeof(struct cmsghdr)) {
1415 return FALSE;
1416 }
1417
1418 if ((uint8_t *)control->m_data >= (uint8_t *)cmsg + cmsg->cmsg_len) {
1419 return FALSE;
1420 }
1421
1422 if ((uint8_t *)control->m_data + control->m_len <
1423 (uint8_t *)cmsg + cmsg->cmsg_len) {
1424 return FALSE;
1425 }
1426
1427 return TRUE;
1428}
1429
1430static int
1431sbappendcontrol_internal(struct sockbuf *sb, struct mbuf *m0,
1432 struct mbuf *control)
1433{
1434 struct mbuf *m, *mlast, *n;
1435 int space = 0;
1436
1437 if (control == 0) {
1438 panic("sbappendcontrol");
1439 }
1440
1441 for (m = control;; m = m->m_next) {
1442 space += m->m_len;
1443 if (m->m_next == 0) {
1444 break;
1445 }
1446 }
1447 n = m; /* save pointer to last control buffer */
1448 for (m = m0; m; m = m->m_next) {
1449 space += m->m_len;
1450 }
1451 if (space > sbspace(sb) && !(sb->sb_flags & SB_UNIX)) {
1452 return 0;
1453 }
1454 n->m_next = m0; /* concatenate data to control */
1455 SBLASTRECORDCHK(sb, "sbappendcontrol 1");
1456
1457 for (m = control; m->m_next != NULL; m = m->m_next) {
1458 sballoc(sb, m);
1459 }
1460 sballoc(sb, m);
1461 mlast = m;
1462
1463 if (sb->sb_lastrecord != NULL) {
1464 sb->sb_lastrecord->m_nextpkt = control;
1465 } else {
1466 sb->sb_mb = control;
1467 }
1468 sb->sb_lastrecord = control;
1469 sb->sb_mbtail = mlast;
1470
1471 SBLASTMBUFCHK(sb, __func__);
1472 SBLASTRECORDCHK(sb, "sbappendcontrol 2");
1473 return 1;
1474}
1475
1476int
1477sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control,
1478 int *error_out)
1479{
1480 struct soflow_hash_entry *dgram_flow_entry = NULL;
1481 struct socket *so = sb->sb_so;
1482 int result = 0;
1483 boolean_t sb_unix = (sb->sb_flags & SB_UNIX);
1484
1485 if (error_out) {
1486 *error_out = 0;
1487 }
1488
1489 if (sb->sb_flags & SB_DROP) {
1490 if (m0 != NULL) {
1491 m_freem(m0);
1492 }
1493 if (control != NULL && !sb_unix) {
1494 m_freem(control);
1495 }
1496 if (error_out != NULL) {
1497 *error_out = EINVAL;
1498 }
1499 return 0;
1500 }
1501
1502 if (SOCK_DOM(sb->sb_so) == PF_INET || SOCK_DOM(sb->sb_so) == PF_INET6) {
1503 if (NEED_DGRAM_FLOW_TRACKING(so)) {
1504 dgram_flow_entry = soflow_get_flow(so, NULL, NULL, control, m0 != NULL ? m_length(m0) : 0, false, (m0 != NULL && m0->m_pkthdr.rcvif) ? m0->m_pkthdr.rcvif->if_index : 0);
1505 }
1506
1507 if (sb->sb_flags & SB_RECV && !(m0 && m0->m_flags & M_SKIPCFIL)) {
1508 int error;
1509
1510 error = sflt_data_in(so: sb->sb_so, NULL, data: &m0, control: &control, flags: 0);
1511 SBLASTRECORDCHK(sb, __func__);
1512
1513#if CONTENT_FILTER
1514 if (error == 0) {
1515 error = cfil_sock_data_in(so: sb->sb_so, NULL, data: m0, control,
1516 flags: 0, dgram_flow_entry);
1517 }
1518#endif /* CONTENT_FILTER */
1519
1520 if (error) {
1521 if (error != EJUSTRETURN) {
1522 if (m0) {
1523 m_freem(m0);
1524 }
1525 if (control != NULL && !sb_unix) {
1526 m_freem(control);
1527 }
1528 if (error_out) {
1529 *error_out = error;
1530 }
1531 }
1532 if (dgram_flow_entry != NULL) {
1533 soflow_free_flow(dgram_flow_entry);
1534 }
1535 return 0;
1536 }
1537 } else if (m0) {
1538 m0->m_flags &= ~M_SKIPCFIL;
1539 }
1540
1541 if (dgram_flow_entry != NULL) {
1542 soflow_free_flow(dgram_flow_entry);
1543 }
1544 }
1545
1546 result = sbappendcontrol_internal(sb, m0, control);
1547 if (result == 0) {
1548 if (m0) {
1549 m_freem(m0);
1550 }
1551 if (control != NULL && !sb_unix) {
1552 m_freem(control);
1553 }
1554 if (error_out) {
1555 *error_out = ENOBUFS;
1556 }
1557 }
1558
1559 return result;
1560}
1561
1562/*
1563 * TCP streams have Multipath TCP support or are regular TCP sockets.
1564 */
1565int
1566sbappendstream_rcvdemux(struct socket *so, struct mbuf *m)
1567{
1568 int ret = 0;
1569
1570 if ((m != NULL) &&
1571 m_pktlen(m) <= 0 &&
1572 !((so->so_flags & SOF_MP_SUBFLOW) &&
1573 (m->m_flags & M_PKTHDR) &&
1574 (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN))) {
1575 m_freem(m);
1576 return ret;
1577 }
1578
1579#if MPTCP
1580 if (so->so_flags & SOF_MP_SUBFLOW) {
1581 return sbappendmptcpstream_rcv(sb: &so->so_rcv, m);
1582 } else
1583#endif /* MPTCP */
1584 {
1585 return sbappendstream(sb: &so->so_rcv, m);
1586 }
1587}
1588
1589#if MPTCP
1590int
1591sbappendmptcpstream_rcv(struct sockbuf *sb, struct mbuf *m)
1592{
1593 struct socket *so = sb->sb_so;
1594
1595 VERIFY(m == NULL || (m->m_flags & M_PKTHDR));
1596 /* SB_NOCOMPRESS must be set prevent loss of M_PKTHDR data */
1597 VERIFY((sb->sb_flags & (SB_RECV | SB_NOCOMPRESS)) ==
1598 (SB_RECV | SB_NOCOMPRESS));
1599
1600 if (m == NULL || m_pktlen(m) == 0 || (sb->sb_flags & SB_DROP) ||
1601 (so->so_state & SS_CANTRCVMORE)) {
1602 if (m && (m->m_flags & M_PKTHDR) &&
1603 m_pktlen(m) == 0 &&
1604 (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)) {
1605 mptcp_input(tptomptp(sototcpcb(so))->mpt_mpte, m);
1606 return 1;
1607 } else if (m != NULL) {
1608 m_freem(m);
1609 }
1610 return 0;
1611 }
1612 /* the socket is not closed, so SOF_MP_SUBFLOW must be set */
1613 VERIFY(so->so_flags & SOF_MP_SUBFLOW);
1614
1615 if (m->m_nextpkt != NULL || (sb->sb_mb != sb->sb_lastrecord)) {
1616 panic("%s: nexpkt %p || mb %p != lastrecord %p", __func__,
1617 m->m_nextpkt, sb->sb_mb, sb->sb_lastrecord);
1618 /* NOTREACHED */
1619 }
1620
1621 SBLASTMBUFCHK(sb, __func__);
1622
1623 /* No filter support (SB_RECV) on mptcp subflow sockets */
1624
1625 sbcompress(sb, m, sb->sb_mbtail);
1626 sb->sb_lastrecord = sb->sb_mb;
1627 SBLASTRECORDCHK(sb, __func__);
1628 return 1;
1629}
1630#endif /* MPTCP */
1631
1632/*
1633 * Compress mbuf chain m into the socket
1634 * buffer sb following mbuf n. If n
1635 * is null, the buffer is presumed empty.
1636 */
1637static inline void
1638sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n)
1639{
1640 int eor = 0, compress = (!(sb->sb_flags & SB_NOCOMPRESS));
1641 struct mbuf *o;
1642
1643 if (m == NULL) {
1644 /* There is nothing to compress; just update the tail */
1645 for (; n->m_next != NULL; n = n->m_next) {
1646 ;
1647 }
1648 sb->sb_mbtail = n;
1649 goto done;
1650 }
1651
1652 while (m != NULL) {
1653 eor |= m->m_flags & M_EOR;
1654 if (compress && m->m_len == 0 && (eor == 0 ||
1655 (((o = m->m_next) || (o = n)) && o->m_type == m->m_type))) {
1656 if (sb->sb_lastrecord == m) {
1657 sb->sb_lastrecord = m->m_next;
1658 }
1659 m = m_free(m);
1660 continue;
1661 }
1662 if (compress && n != NULL && (n->m_flags & M_EOR) == 0 &&
1663#ifndef __APPLE__
1664 M_WRITABLE(n) &&
1665#endif
1666 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */
1667 m->m_len <= M_TRAILINGSPACE(n) &&
1668 n->m_type == m->m_type) {
1669 bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len,
1670 n: (unsigned)m->m_len);
1671 n->m_len += m->m_len;
1672 sb->sb_cc += m->m_len;
1673 if (!m_has_mtype(m, mtype_flags: MTF_DATA | MTF_HEADER | MTF_OOBDATA)) {
1674 /* XXX: Probably don't need */
1675 sb->sb_ctl += m->m_len;
1676 }
1677
1678 /* update send byte count */
1679 if (sb->sb_flags & SB_SNDBYTE_CNT) {
1680 inp_incr_sndbytes_total(sb->sb_so,
1681 m->m_len);
1682 inp_incr_sndbytes_unsent(sb->sb_so,
1683 m->m_len);
1684 }
1685 m = m_free(m);
1686 continue;
1687 }
1688 if (n != NULL) {
1689 n->m_next = m;
1690 } else {
1691 sb->sb_mb = m;
1692 }
1693 sb->sb_mbtail = m;
1694 sballoc(sb, m);
1695 n = m;
1696 m->m_flags &= ~M_EOR;
1697 m = m->m_next;
1698 n->m_next = NULL;
1699 }
1700 if (eor != 0) {
1701 if (n != NULL) {
1702 n->m_flags |= M_EOR;
1703 } else {
1704 printf("semi-panic: sbcompress\n");
1705 }
1706 }
1707done:
1708 SBLASTMBUFCHK(sb, __func__);
1709}
1710
1711void
1712sb_empty_assert(struct sockbuf *sb, const char *where)
1713{
1714 if (!(sb->sb_cc == 0 && sb->sb_mb == NULL && sb->sb_mbcnt == 0 &&
1715 sb->sb_mbtail == NULL && sb->sb_lastrecord == NULL)) {
1716 panic("%s: sb %p so %p cc %d mbcnt %d mb %p mbtail %p "
1717 "lastrecord %p\n", where, sb, sb->sb_so, sb->sb_cc,
1718 sb->sb_mbcnt, sb->sb_mb, sb->sb_mbtail,
1719 sb->sb_lastrecord);
1720 /* NOTREACHED */
1721 }
1722}
1723
1724/*
1725 * Free all mbufs in a sockbuf.
1726 * Check that all resources are reclaimed.
1727 */
1728void
1729sbflush(struct sockbuf *sb)
1730{
1731 void *lr_saved = __builtin_return_address(0);
1732 struct socket *so = sb->sb_so;
1733
1734 /* so_usecount may be 0 if we get here from sofreelastref() */
1735 if (so == NULL) {
1736 panic("%s: null so, sb=%p sb_flags=0x%x lr=%p",
1737 __func__, sb, sb->sb_flags, lr_saved);
1738 /* NOTREACHED */
1739 } else if (so->so_usecount < 0) {
1740 panic("%s: sb=%p sb_flags=0x%x sb_so=%p usecount=%d lr=%p "
1741 "lrh= %s\n", __func__, sb, sb->sb_flags, so,
1742 so->so_usecount, lr_saved, solockhistory_nr(so));
1743 /* NOTREACHED */
1744 }
1745
1746 /*
1747 * Obtain lock on the socket buffer (SB_LOCK). This is required
1748 * to prevent the socket buffer from being unexpectedly altered
1749 * while it is used by another thread in socket send/receive.
1750 *
1751 * sblock() must not fail here, hence the assertion.
1752 */
1753 (void) sblock(sb, SBL_WAIT | SBL_NOINTR | SBL_IGNDEFUNCT);
1754 VERIFY(sb->sb_flags & SB_LOCK);
1755
1756 while (sb->sb_mbcnt > 0) {
1757 /*
1758 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty:
1759 * we would loop forever. Panic instead.
1760 */
1761 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len)) {
1762 break;
1763 }
1764 sbdrop(sb, len: (int)sb->sb_cc);
1765 }
1766
1767 if (sb->sb_flags & SB_SENDHEAD) {
1768 sb->sb_sendhead = NULL;
1769 }
1770
1771 sb_empty_assert(sb, where: __func__);
1772 sbunlock(sb, TRUE); /* keep socket locked */
1773}
1774
1775/*
1776 * Drop data from (the front of) a sockbuf.
1777 * use m_freem_list to free the mbuf structures
1778 * under a single lock... this is done by pruning
1779 * the top of the tree from the body by keeping track
1780 * of where we get to in the tree and then zeroing the
1781 * two pertinent pointers m_nextpkt and m_next
1782 * the socket buffer is then updated to point at the new
1783 * top of the tree and the pruned area is released via
1784 * m_freem_list.
1785 */
1786void
1787sbdrop(struct sockbuf *sb, int len)
1788{
1789 struct mbuf *m, *free_list, *ml;
1790 struct mbuf *next, *last;
1791
1792 next = (m = sb->sb_mb) ? m->m_nextpkt : 0;
1793#if MPTCP
1794 if (m != NULL && len > 0 && !(sb->sb_flags & SB_RECV) &&
1795 ((sb->sb_so->so_flags & SOF_MP_SUBFLOW) ||
1796 (SOCK_CHECK_DOM(sb->sb_so, PF_MULTIPATH) &&
1797 SOCK_CHECK_PROTO(sb->sb_so, IPPROTO_TCP))) &&
1798 !(sb->sb_so->so_flags1 & SOF1_POST_FALLBACK_SYNC)) {
1799 mptcp_preproc_sbdrop(sb->sb_so, m, (unsigned int)len);
1800 }
1801 if (m != NULL && len > 0 && !(sb->sb_flags & SB_RECV) &&
1802 (sb->sb_so->so_flags & SOF_MP_SUBFLOW) &&
1803 (sb->sb_so->so_flags1 & SOF1_POST_FALLBACK_SYNC)) {
1804 mptcp_fallback_sbdrop(so: sb->sb_so, m, len);
1805 }
1806#endif /* MPTCP */
1807 KERNEL_DEBUG((DBG_FNC_SBDROP | DBG_FUNC_START), sb, len, 0, 0, 0);
1808
1809 free_list = last = m;
1810 ml = (struct mbuf *)0;
1811
1812 if (sb->sb_flags & SB_SENDHEAD) {
1813 sb->sb_sendoff -= MIN(len, sb->sb_sendoff);
1814 }
1815
1816 while (len > 0) {
1817 if (m == NULL) {
1818 if (next == NULL) {
1819 /*
1820 * temporarily replacing this panic with printf
1821 * because it occurs occasionally when closing
1822 * a socket when there is no harm in ignoring
1823 * it. This problem will be investigated
1824 * further.
1825 */
1826 /* panic("sbdrop"); */
1827 printf("sbdrop - count not zero\n");
1828 len = 0;
1829 /*
1830 * zero the counts. if we have no mbufs,
1831 * we have no data (PR-2986815)
1832 */
1833 sb->sb_cc = 0;
1834 sb->sb_mbcnt = 0;
1835 break;
1836 }
1837 m = last = next;
1838 next = m->m_nextpkt;
1839 continue;
1840 }
1841 if (m->m_len > len) {
1842 m->m_len -= len;
1843 m->m_data += len;
1844 sb->sb_cc -= len;
1845 /* update the send byte count */
1846 if (sb->sb_flags & SB_SNDBYTE_CNT) {
1847 inp_decr_sndbytes_total(sb->sb_so, len);
1848 }
1849 if (sb->sb_flags & SB_SENDHEAD) {
1850 if (sb->sb_sendhead == m) {
1851 sb->sb_sendhead = NULL;
1852 }
1853 }
1854 if (!m_has_mtype(m, mtype_flags: MTF_DATA | MTF_HEADER | MTF_OOBDATA)) {
1855 sb->sb_ctl -= len;
1856 }
1857 break;
1858 }
1859 len -= m->m_len;
1860 sbfree(sb, m);
1861
1862 ml = m;
1863 m = m->m_next;
1864 }
1865 while (m && m->m_len == 0) {
1866 sbfree(sb, m);
1867
1868 ml = m;
1869 m = m->m_next;
1870 }
1871 if (ml) {
1872 ml->m_next = (struct mbuf *)0;
1873 last->m_nextpkt = (struct mbuf *)0;
1874 m_freem_list(free_list);
1875 }
1876 if (m) {
1877 sb->sb_mb = m;
1878 m->m_nextpkt = next;
1879 } else {
1880 sb->sb_mb = next;
1881 }
1882
1883 /*
1884 * First part is an inline SB_EMPTY_FIXUP(). Second part
1885 * makes sure sb_lastrecord is up-to-date if we dropped
1886 * part of the last record.
1887 */
1888 m = sb->sb_mb;
1889 if (m == NULL) {
1890 sb->sb_mbtail = NULL;
1891 sb->sb_lastrecord = NULL;
1892 } else if (m->m_nextpkt == NULL) {
1893 sb->sb_lastrecord = m;
1894 }
1895
1896#if CONTENT_FILTER
1897 cfil_sock_buf_update(sb);
1898#endif /* CONTENT_FILTER */
1899
1900 KERNEL_DEBUG((DBG_FNC_SBDROP | DBG_FUNC_END), sb, 0, 0, 0, 0);
1901}
1902
1903/*
1904 * Drop a record off the front of a sockbuf
1905 * and move the next record to the front.
1906 */
1907void
1908sbdroprecord(struct sockbuf *sb)
1909{
1910 struct mbuf *m, *mn;
1911
1912 m = sb->sb_mb;
1913 if (m) {
1914 sb->sb_mb = m->m_nextpkt;
1915 do {
1916 sbfree(sb, m);
1917 MFREE(m, mn);
1918 m = mn;
1919 } while (m);
1920 }
1921 SB_EMPTY_FIXUP(sb);
1922}
1923
1924/*
1925 * Create a "control" mbuf containing the specified data
1926 * with the specified type for presentation on a socket buffer.
1927 */
1928struct mbuf *
1929sbcreatecontrol(caddr_t p, int size, int type, int level)
1930{
1931 struct cmsghdr *cp;
1932 struct mbuf *m;
1933
1934 if (CMSG_SPACE((u_int)size) > MLEN) {
1935 return (struct mbuf *)NULL;
1936 }
1937 if ((m = m_get(M_DONTWAIT, MT_CONTROL)) == NULL) {
1938 return (struct mbuf *)NULL;
1939 }
1940 cp = mtod(m, struct cmsghdr *);
1941 VERIFY(IS_P2ALIGNED(cp, sizeof(u_int32_t)));
1942 /* XXX check size? */
1943 (void) memcpy(CMSG_DATA(cp), src: p, n: size);
1944 m->m_len = (int32_t)CMSG_SPACE(size);
1945 cp->cmsg_len = CMSG_LEN(size);
1946 cp->cmsg_level = level;
1947 cp->cmsg_type = type;
1948 return m;
1949}
1950
1951struct mbuf **
1952sbcreatecontrol_mbuf(caddr_t p, int size, int type, int level, struct mbuf **mp)
1953{
1954 struct mbuf *m;
1955 struct cmsghdr *cp;
1956
1957 if (*mp == NULL) {
1958 *mp = sbcreatecontrol(p, size, type, level);
1959 return mp;
1960 }
1961
1962 if (CMSG_SPACE((u_int)size) + (*mp)->m_len > MLEN) {
1963 mp = &(*mp)->m_next;
1964 *mp = sbcreatecontrol(p, size, type, level);
1965 return mp;
1966 }
1967
1968 m = *mp;
1969
1970 cp = (struct cmsghdr *)(void *)(mtod(m, char *) + m->m_len);
1971 /* CMSG_SPACE ensures 32-bit alignment */
1972 VERIFY(IS_P2ALIGNED(cp, sizeof(u_int32_t)));
1973 m->m_len += (int32_t)CMSG_SPACE(size);
1974
1975 /* XXX check size? */
1976 (void) memcpy(CMSG_DATA(cp), src: p, n: size);
1977 cp->cmsg_len = CMSG_LEN(size);
1978 cp->cmsg_level = level;
1979 cp->cmsg_type = type;
1980
1981 return mp;
1982}
1983
1984
1985/*
1986 * Some routines that return EOPNOTSUPP for entry points that are not
1987 * supported by a protocol. Fill in as needed.
1988 */
1989int
1990pru_abort_notsupp(struct socket *so)
1991{
1992#pragma unused(so)
1993 return EOPNOTSUPP;
1994}
1995
1996int
1997pru_accept_notsupp(struct socket *so, struct sockaddr **nam)
1998{
1999#pragma unused(so, nam)
2000 return EOPNOTSUPP;
2001}
2002
2003int
2004pru_attach_notsupp(struct socket *so, int proto, struct proc *p)
2005{
2006#pragma unused(so, proto, p)
2007 return EOPNOTSUPP;
2008}
2009
2010int
2011pru_bind_notsupp(struct socket *so, struct sockaddr *nam, struct proc *p)
2012{
2013#pragma unused(so, nam, p)
2014 return EOPNOTSUPP;
2015}
2016
2017int
2018pru_connect_notsupp(struct socket *so, struct sockaddr *nam, struct proc *p)
2019{
2020#pragma unused(so, nam, p)
2021 return EOPNOTSUPP;
2022}
2023
2024int
2025pru_connect2_notsupp(struct socket *so1, struct socket *so2)
2026{
2027#pragma unused(so1, so2)
2028 return EOPNOTSUPP;
2029}
2030
2031int
2032pru_connectx_notsupp(struct socket *so, struct sockaddr *src,
2033 struct sockaddr *dst, struct proc *p, uint32_t ifscope,
2034 sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg,
2035 uint32_t arglen, struct uio *uio, user_ssize_t *bytes_written)
2036{
2037#pragma unused(so, src, dst, p, ifscope, aid, pcid, flags, arg, arglen, uio, bytes_written)
2038 return EOPNOTSUPP;
2039}
2040
2041int
2042pru_control_notsupp(struct socket *so, u_long cmd, caddr_t data,
2043 struct ifnet *ifp, struct proc *p)
2044{
2045#pragma unused(so, cmd, data, ifp, p)
2046 return EOPNOTSUPP;
2047}
2048
2049int
2050pru_detach_notsupp(struct socket *so)
2051{
2052#pragma unused(so)
2053 return EOPNOTSUPP;
2054}
2055
2056int
2057pru_disconnect_notsupp(struct socket *so)
2058{
2059#pragma unused(so)
2060 return EOPNOTSUPP;
2061}
2062
2063int
2064pru_disconnectx_notsupp(struct socket *so, sae_associd_t aid, sae_connid_t cid)
2065{
2066#pragma unused(so, aid, cid)
2067 return EOPNOTSUPP;
2068}
2069
2070int
2071pru_listen_notsupp(struct socket *so, struct proc *p)
2072{
2073#pragma unused(so, p)
2074 return EOPNOTSUPP;
2075}
2076
2077int
2078pru_peeraddr_notsupp(struct socket *so, struct sockaddr **nam)
2079{
2080#pragma unused(so, nam)
2081 return EOPNOTSUPP;
2082}
2083
2084int
2085pru_rcvd_notsupp(struct socket *so, int flags)
2086{
2087#pragma unused(so, flags)
2088 return EOPNOTSUPP;
2089}
2090
2091int
2092pru_rcvoob_notsupp(struct socket *so, struct mbuf *m, int flags)
2093{
2094#pragma unused(so, m, flags)
2095 return EOPNOTSUPP;
2096}
2097
2098int
2099pru_send_notsupp(struct socket *so, int flags, struct mbuf *m,
2100 struct sockaddr *addr, struct mbuf *control, struct proc *p)
2101{
2102#pragma unused(so, flags, m, addr, control, p)
2103 return EOPNOTSUPP;
2104}
2105
2106int
2107pru_send_list_notsupp(struct socket *so, struct mbuf *m, u_int *pktcnt,
2108 int flags)
2109{
2110#pragma unused(so, m, pktcnt, flags)
2111 return EOPNOTSUPP;
2112}
2113
2114/*
2115 * This isn't really a ``null'' operation, but it's the default one
2116 * and doesn't do anything destructive.
2117 */
2118int
2119pru_sense_null(struct socket *so, void *ub, int isstat64)
2120{
2121 if (isstat64 != 0) {
2122 struct stat64 *sb64;
2123
2124 sb64 = (struct stat64 *)ub;
2125 sb64->st_blksize = so->so_snd.sb_hiwat;
2126 } else {
2127 struct stat *sb;
2128
2129 sb = (struct stat *)ub;
2130 sb->st_blksize = so->so_snd.sb_hiwat;
2131 }
2132
2133 return 0;
2134}
2135
2136int
2137pru_sosend_notsupp(struct socket *so, struct sockaddr *addr, struct uio *uio,
2138 struct mbuf *top, struct mbuf *control, int flags)
2139{
2140#pragma unused(so, addr, uio, top, control, flags)
2141 return EOPNOTSUPP;
2142}
2143
2144int
2145pru_sosend_list_notsupp(struct socket *so, struct mbuf *m, size_t total_len, u_int *pktcnt, int flags)
2146{
2147#pragma unused(so, m, total_len, pktcnt, flags)
2148 return EOPNOTSUPP;
2149}
2150
2151int
2152pru_soreceive_notsupp(struct socket *so, struct sockaddr **paddr,
2153 struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
2154{
2155#pragma unused(so, paddr, uio, mp0, controlp, flagsp)
2156 return EOPNOTSUPP;
2157}
2158
2159int
2160pru_shutdown_notsupp(struct socket *so)
2161{
2162#pragma unused(so)
2163 return EOPNOTSUPP;
2164}
2165
2166int
2167pru_sockaddr_notsupp(struct socket *so, struct sockaddr **nam)
2168{
2169#pragma unused(so, nam)
2170 return EOPNOTSUPP;
2171}
2172
2173int
2174pru_sopoll_notsupp(struct socket *so, int events, kauth_cred_t cred, void *wql)
2175{
2176#pragma unused(so, events, cred, wql)
2177 return EOPNOTSUPP;
2178}
2179
2180int
2181pru_socheckopt_null(struct socket *so, struct sockopt *sopt)
2182{
2183#pragma unused(so, sopt)
2184 /*
2185 * Allow all options for set/get by default.
2186 */
2187 return 0;
2188}
2189
2190static int
2191pru_preconnect_null(struct socket *so)
2192{
2193#pragma unused(so)
2194 return 0;
2195}
2196
2197static int
2198pru_defunct_null(struct socket *so)
2199{
2200#pragma unused(so)
2201 return 0;
2202}
2203
2204
2205void
2206pru_sanitize(struct pr_usrreqs *pru)
2207{
2208#define DEFAULT(foo, bar) if ((foo) == NULL) (foo) = (bar)
2209 DEFAULT(pru->pru_abort, pru_abort_notsupp);
2210 DEFAULT(pru->pru_accept, pru_accept_notsupp);
2211 DEFAULT(pru->pru_attach, pru_attach_notsupp);
2212 DEFAULT(pru->pru_bind, pru_bind_notsupp);
2213 DEFAULT(pru->pru_connect, pru_connect_notsupp);
2214 DEFAULT(pru->pru_connect2, pru_connect2_notsupp);
2215 DEFAULT(pru->pru_connectx, pru_connectx_notsupp);
2216 DEFAULT(pru->pru_control, pru_control_notsupp);
2217 DEFAULT(pru->pru_detach, pru_detach_notsupp);
2218 DEFAULT(pru->pru_disconnect, pru_disconnect_notsupp);
2219 DEFAULT(pru->pru_disconnectx, pru_disconnectx_notsupp);
2220 DEFAULT(pru->pru_listen, pru_listen_notsupp);
2221 DEFAULT(pru->pru_peeraddr, pru_peeraddr_notsupp);
2222 DEFAULT(pru->pru_rcvd, pru_rcvd_notsupp);
2223 DEFAULT(pru->pru_rcvoob, pru_rcvoob_notsupp);
2224 DEFAULT(pru->pru_send, pru_send_notsupp);
2225 DEFAULT(pru->pru_send_list, pru_send_list_notsupp);
2226 DEFAULT(pru->pru_sense, pru_sense_null);
2227 DEFAULT(pru->pru_shutdown, pru_shutdown_notsupp);
2228 DEFAULT(pru->pru_sockaddr, pru_sockaddr_notsupp);
2229 DEFAULT(pru->pru_sopoll, pru_sopoll_notsupp);
2230 DEFAULT(pru->pru_soreceive, pru_soreceive_notsupp);
2231 DEFAULT(pru->pru_sosend, pru_sosend_notsupp);
2232 DEFAULT(pru->pru_sosend_list, pru_sosend_list_notsupp);
2233 DEFAULT(pru->pru_socheckopt, pru_socheckopt_null);
2234 DEFAULT(pru->pru_preconnect, pru_preconnect_null);
2235 DEFAULT(pru->pru_defunct, pru_defunct_null);
2236#undef DEFAULT
2237}
2238
2239/*
2240 * The following are macros on BSD and functions on Darwin
2241 */
2242
2243/*
2244 * Do we need to notify the other side when I/O is possible?
2245 */
2246
2247int
2248sb_notify(struct sockbuf *sb)
2249{
2250 return sb->sb_waiters > 0 ||
2251 (sb->sb_flags & (SB_SEL | SB_ASYNC | SB_UPCALL | SB_KNOTE));
2252}
2253
2254/*
2255 * How much space is there in a socket buffer (so->so_snd or so->so_rcv)?
2256 * This is problematical if the fields are unsigned, as the space might
2257 * still be negative (cc > hiwat or mbcnt > mbmax). Should detect
2258 * overflow and return 0.
2259 */
2260int
2261sbspace(struct sockbuf *sb)
2262{
2263 int pending = 0;
2264 int space;
2265
2266 if (sb->sb_flags & SB_KCTL) {
2267 space = (int)(sb->sb_hiwat - sb->sb_cc);
2268 } else {
2269 space = imin(a: (int)(sb->sb_hiwat - sb->sb_cc),
2270 b: (int)(sb->sb_mbmax - sb->sb_mbcnt));
2271 }
2272 if (sb->sb_preconn_hiwat != 0) {
2273 space = imin(a: (int)(sb->sb_preconn_hiwat - sb->sb_cc), b: space);
2274 }
2275
2276 if (space < 0) {
2277 space = 0;
2278 }
2279
2280 /* Compensate for data being processed by content filters */
2281#if CONTENT_FILTER
2282 pending = cfil_sock_data_space(sb);
2283#endif /* CONTENT_FILTER */
2284 if (pending > space) {
2285 space = 0;
2286 } else {
2287 space -= pending;
2288 }
2289
2290 return space;
2291}
2292
2293/* do we have to send all at once on a socket? */
2294int
2295sosendallatonce(struct socket *so)
2296{
2297 return so->so_proto->pr_flags & PR_ATOMIC;
2298}
2299
2300/* can we read something from so? */
2301int
2302soreadable(struct socket *so)
2303{
2304 return so->so_rcv.sb_cc >= so->so_rcv.sb_lowat ||
2305 ((so->so_state & SS_CANTRCVMORE)
2306#if CONTENT_FILTER
2307 && cfil_sock_data_pending(sb: &so->so_rcv) == 0
2308#endif /* CONTENT_FILTER */
2309 ) ||
2310 so->so_comp.tqh_first || so->so_error;
2311}
2312
2313/* can we write something to so? */
2314
2315int
2316sowriteable(struct socket *so)
2317{
2318 if ((so->so_state & SS_CANTSENDMORE) ||
2319 so->so_error > 0) {
2320 return 1;
2321 }
2322 if (so_wait_for_if_feedback(so) || !socanwrite(so)) {
2323 return 0;
2324 }
2325 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2326 return 1;
2327 }
2328
2329 int64_t data = sbspace(sb: &so->so_snd);
2330 int64_t lowat = so->so_snd.sb_lowat;
2331 /*
2332 * Deal with connected UNIX domain sockets which
2333 * rely on the fact that the sender's socket buffer is
2334 * actually the receiver's socket buffer.
2335 */
2336 if (SOCK_DOM(so) == PF_LOCAL) {
2337 struct unpcb *unp = sotounpcb(so);
2338 if (unp != NULL && unp->unp_conn != NULL &&
2339 unp->unp_conn->unp_socket != NULL) {
2340 struct socket *so2 = unp->unp_conn->unp_socket;
2341 /*
2342 * At this point we know that `so' is locked
2343 * and that `unp_conn` isn't going to change.
2344 * However, we don't lock `so2` because doing so
2345 * may require unlocking `so'
2346 * (see unp_get_locks_in_order()).
2347 *
2348 * Two cases can happen:
2349 *
2350 * 1) we return 1 and tell the application that
2351 * it can write. Meanwhile, another thread
2352 * fills up the socket buffer. This will either
2353 * lead to a blocking send or EWOULDBLOCK
2354 * which the application should deal with.
2355 * 2) we return 0 and tell the application that
2356 * the socket is not writable. Meanwhile,
2357 * another thread depletes the receive socket
2358 * buffer. In this case the application will
2359 * be woken up by sb_notify().
2360 *
2361 * MIN() is required because otherwise sosendcheck()
2362 * may return EWOULDBLOCK since it only considers
2363 * so->so_snd.
2364 */
2365 data = MIN(data, sbspace(&so2->so_rcv));
2366 }
2367 }
2368
2369 if (data >= lowat) {
2370 if (so->so_flags & SOF_NOTSENT_LOWAT) {
2371 if ((SOCK_DOM(so) == PF_INET6 ||
2372 SOCK_DOM(so) == PF_INET) &&
2373 so->so_type == SOCK_STREAM) {
2374 return tcp_notsent_lowat_check(so);
2375 }
2376#if MPTCP
2377 else if ((SOCK_DOM(so) == PF_MULTIPATH) &&
2378 (SOCK_PROTO(so) == IPPROTO_TCP)) {
2379 return mptcp_notsent_lowat_check(so);
2380 }
2381#endif
2382 else {
2383 return 1;
2384 }
2385 } else {
2386 return 1;
2387 }
2388 }
2389 return 0;
2390}
2391
2392/* adjust counters in sb reflecting allocation of m */
2393
2394void
2395sballoc(struct sockbuf *sb, struct mbuf *m)
2396{
2397 sb->sb_cc += m->m_len;
2398 if (!m_has_mtype(m, mtype_flags: MTF_DATA | MTF_HEADER | MTF_OOBDATA)) {
2399 sb->sb_ctl += m->m_len;
2400 }
2401 sb->sb_mbcnt += _MSIZE;
2402
2403 if (m->m_flags & M_EXT) {
2404 sb->sb_mbcnt += m->m_ext.ext_size;
2405 }
2406
2407 /*
2408 * If data is being added to the send socket buffer,
2409 * update the send byte count
2410 */
2411 if (sb->sb_flags & SB_SNDBYTE_CNT) {
2412 inp_incr_sndbytes_total(sb->sb_so, m->m_len);
2413 inp_incr_sndbytes_unsent(sb->sb_so, m->m_len);
2414 }
2415}
2416
2417/* adjust counters in sb reflecting freeing of m */
2418void
2419sbfree(struct sockbuf *sb, struct mbuf *m)
2420{
2421 sb->sb_cc -= m->m_len;
2422 if (!m_has_mtype(m, mtype_flags: MTF_DATA | MTF_HEADER | MTF_OOBDATA)) {
2423 sb->sb_ctl -= m->m_len;
2424 }
2425 sb->sb_mbcnt -= _MSIZE;
2426 if (m->m_flags & M_EXT) {
2427 sb->sb_mbcnt -= m->m_ext.ext_size;
2428 }
2429
2430 /*
2431 * If data is being removed from the send socket buffer,
2432 * update the send byte count
2433 */
2434 if (sb->sb_flags & SB_SNDBYTE_CNT) {
2435 inp_decr_sndbytes_total(sb->sb_so, m->m_len);
2436 }
2437
2438 if (sb->sb_flags & SB_SENDHEAD) {
2439 if (m == sb->sb_sendhead) {
2440 sb->sb_sendhead = NULL;
2441 }
2442 }
2443}
2444
2445/*
2446 * Set lock on sockbuf sb; sleep if lock is already held.
2447 * Unless SB_NOINTR is set on sockbuf, sleep is interruptible.
2448 * Returns error without lock if sleep is interrupted.
2449 */
2450int
2451sblock(struct sockbuf *sb, uint32_t flags)
2452{
2453 boolean_t nointr = ((sb->sb_flags & SB_NOINTR) || (flags & SBL_NOINTR));
2454 void *lr_saved = __builtin_return_address(0);
2455 struct socket *so = sb->sb_so;
2456 void * wchan;
2457 int error = 0;
2458 thread_t tp = current_thread();
2459
2460 VERIFY((flags & SBL_VALID) == flags);
2461
2462 /* so_usecount may be 0 if we get here from sofreelastref() */
2463 if (so == NULL) {
2464 panic("%s: null so, sb=%p sb_flags=0x%x lr=%p",
2465 __func__, sb, sb->sb_flags, lr_saved);
2466 /* NOTREACHED */
2467 } else if (so->so_usecount < 0) {
2468 panic("%s: sb=%p sb_flags=0x%x sb_so=%p usecount=%d lr=%p "
2469 "lrh= %s\n", __func__, sb, sb->sb_flags, so,
2470 so->so_usecount, lr_saved, solockhistory_nr(so));
2471 /* NOTREACHED */
2472 }
2473
2474 /*
2475 * The content filter thread must hold the sockbuf lock
2476 */
2477 if ((so->so_flags & SOF_CONTENT_FILTER) && sb->sb_cfil_thread == tp) {
2478 /*
2479 * Don't panic if we are defunct because SB_LOCK has
2480 * been cleared by sodefunct()
2481 */
2482 if (!(so->so_flags & SOF_DEFUNCT) && !(sb->sb_flags & SB_LOCK)) {
2483 panic("%s: SB_LOCK not held for %p",
2484 __func__, sb);
2485 }
2486
2487 /* Keep the sockbuf locked */
2488 return 0;
2489 }
2490
2491 if ((sb->sb_flags & SB_LOCK) && !(flags & SBL_WAIT)) {
2492 return EWOULDBLOCK;
2493 }
2494 /*
2495 * We may get here from sorflush(), in which case "sb" may not
2496 * point to the real socket buffer. Use the actual socket buffer
2497 * address from the socket instead.
2498 */
2499 wchan = (sb->sb_flags & SB_RECV) ?
2500 &so->so_rcv.sb_flags : &so->so_snd.sb_flags;
2501
2502 /*
2503 * A content filter thread has exclusive access to the sockbuf
2504 * until it clears the
2505 */
2506 while ((sb->sb_flags & SB_LOCK) ||
2507 ((so->so_flags & SOF_CONTENT_FILTER) &&
2508 sb->sb_cfil_thread != NULL)) {
2509 lck_mtx_t *mutex_held;
2510
2511 /*
2512 * XXX: This code should be moved up above outside of this loop;
2513 * however, we may get here as part of sofreelastref(), and
2514 * at that time pr_getlock() may no longer be able to return
2515 * us the lock. This will be fixed in future.
2516 */
2517 if (so->so_proto->pr_getlock != NULL) {
2518 mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK);
2519 } else {
2520 mutex_held = so->so_proto->pr_domain->dom_mtx;
2521 }
2522
2523 LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
2524
2525 sb->sb_wantlock++;
2526 VERIFY(sb->sb_wantlock != 0);
2527
2528 error = msleep(chan: wchan, mtx: mutex_held,
2529 pri: nointr ? PSOCK : PSOCK | PCATCH,
2530 wmesg: nointr ? "sb_lock_nointr" : "sb_lock", NULL);
2531
2532 VERIFY(sb->sb_wantlock != 0);
2533 sb->sb_wantlock--;
2534
2535 if (error == 0 && (so->so_flags & SOF_DEFUNCT) &&
2536 !(flags & SBL_IGNDEFUNCT)) {
2537 error = EBADF;
2538 SODEFUNCTLOG("%s[%d, %s]: defunct so 0x%llu [%d,%d] "
2539 "(%d)\n", __func__, proc_selfpid(),
2540 proc_best_name(current_proc()),
2541 so->so_gencnt,
2542 SOCK_DOM(so), SOCK_TYPE(so), error);
2543 }
2544
2545 if (error != 0) {
2546 return error;
2547 }
2548 }
2549 sb->sb_flags |= SB_LOCK;
2550 return 0;
2551}
2552
2553/*
2554 * Release lock on sockbuf sb
2555 */
2556void
2557sbunlock(struct sockbuf *sb, boolean_t keeplocked)
2558{
2559 void *lr_saved = __builtin_return_address(0);
2560 struct socket *so = sb->sb_so;
2561 thread_t tp = current_thread();
2562
2563 /* so_usecount may be 0 if we get here from sofreelastref() */
2564 if (so == NULL) {
2565 panic("%s: null so, sb=%p sb_flags=0x%x lr=%p",
2566 __func__, sb, sb->sb_flags, lr_saved);
2567 /* NOTREACHED */
2568 } else if (so->so_usecount < 0) {
2569 panic("%s: sb=%p sb_flags=0x%x sb_so=%p usecount=%d lr=%p "
2570 "lrh= %s\n", __func__, sb, sb->sb_flags, so,
2571 so->so_usecount, lr_saved, solockhistory_nr(so));
2572 /* NOTREACHED */
2573 }
2574
2575 /*
2576 * The content filter thread must hold the sockbuf lock
2577 */
2578 if ((so->so_flags & SOF_CONTENT_FILTER) && sb->sb_cfil_thread == tp) {
2579 /*
2580 * Don't panic if we are defunct because SB_LOCK has
2581 * been cleared by sodefunct()
2582 */
2583 if (!(so->so_flags & SOF_DEFUNCT) &&
2584 !(sb->sb_flags & SB_LOCK) &&
2585 !(so->so_state & SS_DEFUNCT) &&
2586 !(so->so_flags1 & SOF1_DEFUNCTINPROG)) {
2587 panic("%s: SB_LOCK not held for %p",
2588 __func__, sb);
2589 }
2590 /* Keep the sockbuf locked and proceed */
2591 } else {
2592 VERIFY((sb->sb_flags & SB_LOCK) ||
2593 (so->so_state & SS_DEFUNCT) ||
2594 (so->so_flags1 & SOF1_DEFUNCTINPROG));
2595
2596 sb->sb_flags &= ~SB_LOCK;
2597
2598 if (sb->sb_wantlock > 0) {
2599 /*
2600 * We may get here from sorflush(), in which case "sb"
2601 * may not point to the real socket buffer. Use the
2602 * actual socket buffer address from the socket instead.
2603 */
2604 wakeup(chan: (sb->sb_flags & SB_RECV) ? &so->so_rcv.sb_flags :
2605 &so->so_snd.sb_flags);
2606 }
2607 }
2608
2609 if (!keeplocked) { /* unlock on exit */
2610 if (so->so_flags & SOF_MP_SUBFLOW || SOCK_DOM(so) == PF_MULTIPATH) {
2611 (*so->so_proto->pr_unlock)(so, 1, lr_saved);
2612 } else {
2613 lck_mtx_t *mutex_held;
2614
2615 if (so->so_proto->pr_getlock != NULL) {
2616 mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK);
2617 } else {
2618 mutex_held = so->so_proto->pr_domain->dom_mtx;
2619 }
2620
2621 LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
2622
2623 VERIFY(so->so_usecount > 0);
2624 so->so_usecount--;
2625 so->unlock_lr[so->next_unlock_lr] = lr_saved;
2626 so->next_unlock_lr = (so->next_unlock_lr + 1) % SO_LCKDBG_MAX;
2627 lck_mtx_unlock(lck: mutex_held);
2628 }
2629 }
2630}
2631
2632void
2633sorwakeup(struct socket *so)
2634{
2635 if (sb_notify(sb: &so->so_rcv)) {
2636 sowakeup(so, sb: &so->so_rcv, NULL);
2637 }
2638}
2639
2640void
2641sowwakeup(struct socket *so)
2642{
2643 if (sb_notify(sb: &so->so_snd)) {
2644 sowakeup(so, sb: &so->so_snd, NULL);
2645 }
2646}
2647
2648static void
2649soevupcall(struct socket *so, uint32_t hint)
2650{
2651 if (so->so_event != NULL) {
2652 caddr_t so_eventarg = so->so_eventarg;
2653
2654 hint &= so->so_eventmask;
2655 if (hint != 0) {
2656 so->so_event(so, so_eventarg, hint);
2657 }
2658 }
2659}
2660
2661void
2662soevent(struct socket *so, uint32_t hint)
2663{
2664 if (net_wake_pkt_debug > 0 && (hint & SO_FILT_HINT_WAKE_PKT)) {
2665 os_log(OS_LOG_DEFAULT, "%s: SO_FILT_HINT_WAKE_PKT so %p",
2666 __func__, so);
2667 }
2668
2669 if (so->so_flags & SOF_KNOTE) {
2670 KNOTE(&so->so_klist, hint);
2671 }
2672
2673 soevupcall(so, hint);
2674
2675 /*
2676 * Don't post an event if this a subflow socket or
2677 * the app has opted out of using cellular interface
2678 */
2679 if ((hint & SO_FILT_HINT_IFDENIED) &&
2680 !(so->so_flags & SOF_MP_SUBFLOW) &&
2681 !(so->so_restrictions & SO_RESTRICT_DENY_CELLULAR) &&
2682 !(so->so_restrictions & SO_RESTRICT_DENY_EXPENSIVE) &&
2683 !(so->so_restrictions & SO_RESTRICT_DENY_CONSTRAINED)) {
2684 soevent_ifdenied(so);
2685 }
2686}
2687
2688static void
2689soevent_ifdenied(struct socket *so)
2690{
2691 struct kev_netpolicy_ifdenied ev_ifdenied;
2692
2693 bzero(s: &ev_ifdenied, n: sizeof(ev_ifdenied));
2694 /*
2695 * The event consumer is interested about the effective {upid,pid,uuid}
2696 * info which can be different than the those related to the process
2697 * that recently performed a system call on the socket, i.e. when the
2698 * socket is delegated.
2699 */
2700 if (so->so_flags & SOF_DELEGATED) {
2701 ev_ifdenied.ev_data.eupid = so->e_upid;
2702 ev_ifdenied.ev_data.epid = so->e_pid;
2703 uuid_copy(dst: ev_ifdenied.ev_data.euuid, src: so->e_uuid);
2704 } else {
2705 ev_ifdenied.ev_data.eupid = so->last_upid;
2706 ev_ifdenied.ev_data.epid = so->last_pid;
2707 uuid_copy(dst: ev_ifdenied.ev_data.euuid, src: so->last_uuid);
2708 }
2709
2710 if (++so->so_ifdenied_notifies > 1) {
2711 /*
2712 * Allow for at most one kernel event to be generated per
2713 * socket; so_ifdenied_notifies is reset upon changes in
2714 * the UUID policy. See comments in inp_update_policy.
2715 */
2716 if (net_io_policy_log) {
2717 uuid_string_t buf;
2718
2719 uuid_unparse(uu: ev_ifdenied.ev_data.euuid, out: buf);
2720 log(LOG_DEBUG, "%s[%d]: so %llu [%d,%d] epid %llu "
2721 "euuid %s%s has %d redundant events supressed\n",
2722 __func__, so->last_pid,
2723 so->so_gencnt, SOCK_DOM(so),
2724 SOCK_TYPE(so), ev_ifdenied.ev_data.epid, buf,
2725 ((so->so_flags & SOF_DELEGATED) ?
2726 " [delegated]" : ""), so->so_ifdenied_notifies);
2727 }
2728 } else {
2729 if (net_io_policy_log) {
2730 uuid_string_t buf;
2731
2732 uuid_unparse(uu: ev_ifdenied.ev_data.euuid, out: buf);
2733 log(LOG_DEBUG, "%s[%d]: so %llu [%d,%d] epid %llu "
2734 "euuid %s%s event posted\n", __func__,
2735 so->last_pid, so->so_gencnt,
2736 SOCK_DOM(so), SOCK_TYPE(so),
2737 ev_ifdenied.ev_data.epid, buf,
2738 ((so->so_flags & SOF_DELEGATED) ?
2739 " [delegated]" : ""));
2740 }
2741 netpolicy_post_msg(KEV_NETPOLICY_IFDENIED, &ev_ifdenied.ev_data,
2742 sizeof(ev_ifdenied));
2743 }
2744}
2745
2746/*
2747 * Make a copy of a sockaddr in a malloced buffer of type SONAME.
2748 */
2749struct sockaddr *
2750dup_sockaddr(struct sockaddr *sa, int canwait)
2751{
2752 struct sockaddr *sa2;
2753
2754 sa2 = SA(alloc_sockaddr(sa->sa_len, canwait ? Z_WAITOK : Z_NOWAIT));
2755 if (sa2 != NULL) {
2756 SOCKADDR_COPY(sa, sa2, sa->sa_len);
2757 }
2758 return sa2;
2759}
2760
2761void * __header_indexable
2762alloc_sockaddr(size_t size, zalloc_flags_t flags)
2763{
2764 VERIFY((size) <= UINT8_MAX);
2765
2766 __typed_allocators_ignore_push
2767 void * buf = kheap_alloc(KHEAP_SONAME, size, flags | Z_ZERO);
2768 __typed_allocators_ignore_pop
2769 if (buf != NULL) {
2770 struct sockaddr *sa = SA(buf);
2771 sa->sa_len = (uint8_t)size;
2772 }
2773
2774 return buf;
2775}
2776
2777/*
2778 * Create an external-format (``xsocket'') structure using the information
2779 * in the kernel-format socket structure pointed to by so. This is done
2780 * to reduce the spew of irrelevant information over this interface,
2781 * to isolate user code from changes in the kernel structure, and
2782 * potentially to provide information-hiding if we decide that
2783 * some of this information should be hidden from users.
2784 */
2785void
2786sotoxsocket(struct socket *so, struct xsocket *xso)
2787{
2788 xso->xso_len = sizeof(*xso);
2789 xso->xso_so = (_XSOCKET_PTR(struct socket *))VM_KERNEL_ADDRHASH(so);
2790 xso->so_type = so->so_type;
2791 xso->so_options = (short)(so->so_options & 0xffff);
2792 xso->so_linger = so->so_linger;
2793 xso->so_state = so->so_state;
2794 xso->so_pcb = (_XSOCKET_PTR(caddr_t))VM_KERNEL_ADDRHASH(so->so_pcb);
2795 if (so->so_proto) {
2796 xso->xso_protocol = SOCK_PROTO(so);
2797 xso->xso_family = SOCK_DOM(so);
2798 } else {
2799 xso->xso_protocol = xso->xso_family = 0;
2800 }
2801 xso->so_qlen = so->so_qlen;
2802 xso->so_incqlen = so->so_incqlen;
2803 xso->so_qlimit = so->so_qlimit;
2804 xso->so_timeo = so->so_timeo;
2805 xso->so_error = so->so_error;
2806 xso->so_pgid = so->so_pgid;
2807 xso->so_oobmark = so->so_oobmark;
2808 sbtoxsockbuf(sb: &so->so_snd, xsb: &xso->so_snd);
2809 sbtoxsockbuf(sb: &so->so_rcv, xsb: &xso->so_rcv);
2810 xso->so_uid = kauth_cred_getuid(cred: so->so_cred);
2811}
2812
2813
2814#if XNU_TARGET_OS_OSX
2815
2816void
2817sotoxsocket64(struct socket *so, struct xsocket64 *xso)
2818{
2819 xso->xso_len = sizeof(*xso);
2820 xso->xso_so = (u_int64_t)VM_KERNEL_ADDRHASH(so);
2821 xso->so_type = so->so_type;
2822 xso->so_options = (short)(so->so_options & 0xffff);
2823 xso->so_linger = so->so_linger;
2824 xso->so_state = so->so_state;
2825 xso->so_pcb = (u_int64_t)VM_KERNEL_ADDRHASH(so->so_pcb);
2826 if (so->so_proto) {
2827 xso->xso_protocol = SOCK_PROTO(so);
2828 xso->xso_family = SOCK_DOM(so);
2829 } else {
2830 xso->xso_protocol = xso->xso_family = 0;
2831 }
2832 xso->so_qlen = so->so_qlen;
2833 xso->so_incqlen = so->so_incqlen;
2834 xso->so_qlimit = so->so_qlimit;
2835 xso->so_timeo = so->so_timeo;
2836 xso->so_error = so->so_error;
2837 xso->so_pgid = so->so_pgid;
2838 xso->so_oobmark = so->so_oobmark;
2839 sbtoxsockbuf(sb: &so->so_snd, xsb: &xso->so_snd);
2840 sbtoxsockbuf(sb: &so->so_rcv, xsb: &xso->so_rcv);
2841 xso->so_uid = kauth_cred_getuid(cred: so->so_cred);
2842}
2843
2844#endif /* XNU_TARGET_OS_OSX */
2845
2846/*
2847 * This does the same for sockbufs. Note that the xsockbuf structure,
2848 * since it is always embedded in a socket, does not include a self
2849 * pointer nor a length. We make this entry point public in case
2850 * some other mechanism needs it.
2851 */
2852void
2853sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb)
2854{
2855 xsb->sb_cc = sb->sb_cc;
2856 xsb->sb_hiwat = sb->sb_hiwat;
2857 xsb->sb_mbcnt = sb->sb_mbcnt;
2858 xsb->sb_mbmax = sb->sb_mbmax;
2859 xsb->sb_lowat = sb->sb_lowat;
2860 xsb->sb_flags = (short)sb->sb_flags;
2861 xsb->sb_timeo = (short)
2862 ((sb->sb_timeo.tv_sec * hz) + sb->sb_timeo.tv_usec / tick);
2863 if (xsb->sb_timeo == 0 && sb->sb_timeo.tv_usec != 0) {
2864 xsb->sb_timeo = 1;
2865 }
2866}
2867
2868/*
2869 * Based on the policy set by an all knowing decison maker, throttle sockets
2870 * that either have been marked as belonging to "background" process.
2871 */
2872inline int
2873soisthrottled(struct socket *so)
2874{
2875 return so->so_flags1 & SOF1_TRAFFIC_MGT_SO_BACKGROUND;
2876}
2877
2878inline int
2879soisprivilegedtraffic(struct socket *so)
2880{
2881 return (so->so_flags & SOF_PRIVILEGED_TRAFFIC_CLASS) ? 1 : 0;
2882}
2883
2884inline int
2885soissrcbackground(struct socket *so)
2886{
2887 return (so->so_flags1 & SOF1_TRAFFIC_MGT_SO_BACKGROUND) ||
2888 IS_SO_TC_BACKGROUND(so->so_traffic_class);
2889}
2890
2891inline int
2892soissrcrealtime(struct socket *so)
2893{
2894 return so->so_traffic_class >= SO_TC_AV &&
2895 so->so_traffic_class <= SO_TC_VO;
2896}
2897
2898inline int
2899soissrcbesteffort(struct socket *so)
2900{
2901 return so->so_traffic_class == SO_TC_BE ||
2902 so->so_traffic_class == SO_TC_RD ||
2903 so->so_traffic_class == SO_TC_OAM;
2904}
2905
2906void
2907soclearfastopen(struct socket *so)
2908{
2909 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2910 so->so_flags1 &= ~SOF1_PRECONNECT_DATA;
2911 }
2912
2913 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
2914 so->so_flags1 &= ~SOF1_DATA_IDEMPOTENT;
2915 }
2916}
2917
2918void
2919sonullevent(struct socket *so, void *arg, uint32_t hint)
2920{
2921#pragma unused(so, arg, hint)
2922}
2923
2924/*
2925 * Here is the definition of some of the basic objects in the kern.ipc
2926 * branch of the MIB.
2927 */
2928SYSCTL_NODE(_kern, KERN_IPC, ipc,
2929 CTLFLAG_RW | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, 0, "IPC");
2930
2931/* Check that the maximum socket buffer size is within a range */
2932
2933static int
2934sysctl_sb_max SYSCTL_HANDLER_ARGS
2935{
2936#pragma unused(oidp, arg1, arg2)
2937 u_int32_t new_value;
2938 int changed = 0;
2939 int error = sysctl_io_number(req, bigValue: sb_max, valueSize: sizeof(u_int32_t),
2940 pValue: &new_value, changed: &changed);
2941 if (!error && changed) {
2942 if (new_value > LOW_SB_MAX && new_value <= high_sb_max) {
2943 sb_max = new_value;
2944 } else {
2945 error = ERANGE;
2946 }
2947 }
2948 return error;
2949}
2950
2951SYSCTL_PROC(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf,
2952 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
2953 &sb_max, 0, &sysctl_sb_max, "IU", "Maximum socket buffer size");
2954
2955SYSCTL_INT(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor,
2956 CTLFLAG_RW | CTLFLAG_LOCKED, &sb_efficiency, 0, "");
2957
2958SYSCTL_INT(_kern_ipc, KIPC_NMBCLUSTERS, nmbclusters,
2959 CTLFLAG_RD | CTLFLAG_LOCKED, &nmbclusters, 0, "");
2960
2961SYSCTL_INT(_kern_ipc, OID_AUTO, njcl,
2962 CTLFLAG_RD | CTLFLAG_LOCKED, &njcl, 0, "");
2963
2964SYSCTL_INT(_kern_ipc, OID_AUTO, njclbytes,
2965 CTLFLAG_RD | CTLFLAG_LOCKED, &njclbytes, 0, "");
2966
2967SYSCTL_INT(_kern_ipc, KIPC_SOQLIMITCOMPAT, soqlimitcompat,
2968 CTLFLAG_RW | CTLFLAG_LOCKED, &soqlimitcompat, 1,
2969 "Enable socket queue limit compatibility");
2970
2971/*
2972 * Hack alert -- rdar://33572856
2973 * A loopback test we cannot change was failing because it sets
2974 * SO_SENDTIMEO to 5 seconds and that's also the value
2975 * of the minimum persist timer. Because of the persist timer,
2976 * the connection was not idle for 5 seconds and SO_SNDTIMEO
2977 * was not triggering at 5 seconds causing the test failure.
2978 * As a workaround we check the sysctl soqlencomp the test is already
2979 * setting to set disable auto tuning of the receive buffer.
2980 */
2981
2982extern u_int32_t tcp_do_autorcvbuf;
2983
2984static int
2985sysctl_soqlencomp SYSCTL_HANDLER_ARGS
2986{
2987#pragma unused(oidp, arg1, arg2)
2988 u_int32_t new_value;
2989 int changed = 0;
2990 int error = sysctl_io_number(req, bigValue: soqlencomp, valueSize: sizeof(u_int32_t),
2991 pValue: &new_value, changed: &changed);
2992 if (!error && changed) {
2993 soqlencomp = new_value;
2994 if (new_value != 0) {
2995 tcp_do_autorcvbuf = 0;
2996 tcptv_persmin_val = 6 * TCP_RETRANSHZ;
2997 }
2998 }
2999 return error;
3000}
3001SYSCTL_PROC(_kern_ipc, OID_AUTO, soqlencomp,
3002 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
3003 &soqlencomp, 0, &sysctl_soqlencomp, "IU", "");
3004
3005SYSCTL_NODE(_kern_ipc, OID_AUTO, io_policy, CTLFLAG_RW, 0, "network IO policy");
3006
3007SYSCTL_INT(_kern_ipc_io_policy, OID_AUTO, log, CTLFLAG_RW | CTLFLAG_LOCKED,
3008 &net_io_policy_log, 0, "");
3009
3010#if CONFIG_PROC_UUID_POLICY
3011SYSCTL_INT(_kern_ipc_io_policy, OID_AUTO, uuid, CTLFLAG_RW | CTLFLAG_LOCKED,
3012 &net_io_policy_uuid, 0, "");
3013#endif /* CONFIG_PROC_UUID_POLICY */
3014