uipc_socket2.c source code [xnu/bsd/kern/uipc_socket2.c]

1	/*
2	* Copyright (c) 1998-2020 Apple Inc. All rights reserved.
3	*
4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5	*
6	* This file contains Original Code and/or Modifications of Original Code
7	* as defined in and that are subject to the Apple Public Source License
8	* Version 2.0 (the 'License'). You may not use this file except in
9	* compliance with the License. The rights granted to you under the License
10	* may not be used to create, or enable the creation or redistribution of,
11	* unlawful or unlicensed copies of an Apple operating system, or to
12	* circumvent, violate, or enable the circumvention or violation of, any
13	* terms of an Apple operating system software license agreement.
14	*
15	* Please obtain a copy of the License at
16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
17	*
18	* The Original Code and all software distributed under the License are
19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23	* Please see the License for the specific language governing rights and
24	* limitations under the License.
25	*
26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27	*/
28	/ Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved /
29	/*
30	* Copyright (c) 1982, 1986, 1988, 1990, 1993
31	* The Regents of the University of California. All rights reserved.
32	*
33	* Redistribution and use in source and binary forms, with or without
34	* modification, are permitted provided that the following conditions
35	* are met:
36	* 1. Redistributions of source code must retain the above copyright
37	* notice, this list of conditions and the following disclaimer.
38	* 2. Redistributions in binary form must reproduce the above copyright
39	* notice, this list of conditions and the following disclaimer in the
40	* documentation and/or other materials provided with the distribution.
41	* 3. All advertising materials mentioning features or use of this software
42	* must display the following acknowledgement:
43	* This product includes software developed by the University of
44	* California, Berkeley and its contributors.
45	* 4. Neither the name of the University nor the names of its contributors
46	* may be used to endorse or promote products derived from this software
47	* without specific prior written permission.
48	*
49	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59	* SUCH DAMAGE.
60	*
61	* @(#)uipc_socket2.c 8.1 (Berkeley) 6/10/93
62	*/
63	/*
64	* NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
65	* support for mandatory and extensible security protections. This notice
66	* is included in support of clause 2.2 (b) of the Apple Public License,
67	* Version 2.0.
68	*/
69
70	#include <sys/param.h>
71	#include <sys/systm.h>
72	#include <sys/domain.h>
73	#include <sys/kernel.h>
74	#include <sys/proc_internal.h>
75	#include <sys/kauth.h>
76	#include <sys/malloc.h>
77	#include <sys/mbuf.h>
78	#include <sys/mcache.h>
79	#include <sys/protosw.h>
80	#include <sys/stat.h>
81	#include <sys/socket.h>
82	#include <sys/socketvar.h>
83	#include <sys/signalvar.h>
84	#include <sys/sysctl.h>
85	#include <sys/syslog.h>
86	#include <sys/unpcb.h>
87	#include <sys/ev.h>
88	#include <kern/locks.h>
89	#include <net/route.h>
90	#include <net/content_filter.h>
91	#include <netinet/in.h>
92	#include <netinet/in_pcb.h>
93	#include <netinet/tcp_var.h>
94	#include <sys/kdebug.h>
95	#include <libkern/OSAtomic.h>
96
97	#if CONFIG_MACF
98	#include <security/mac_framework.h>
99	#endif
100
101	#include <mach/vm_param.h>
102
103	#if MPTCP
104	#include <netinet/mptcp_var.h>
105	#endif
106
107	#include <net/sockaddr_utils.h>
108
109	extern uint32_t net_wake_pkt_debug;
110
111	#define DBG_FNC_SBDROP NETDBG_CODE(DBG_NETSOCK, 4)
112	#define DBG_FNC_SBAPPEND NETDBG_CODE(DBG_NETSOCK, 5)
113
114	SYSCTL_DECL(_kern_ipc);
115
116	__private_extern__ u_int32_t net_io_policy_throttle_best_effort = `0`;
117	SYSCTL_INT(_kern_ipc, OID_AUTO, throttle_best_effort,
118	CTLFLAG_RW \| CTLFLAG_LOCKED, &net_io_policy_throttle_best_effort, `0`, "");
119
120	static inline void sbcompress(struct sockbuf , struct* mbuf , struct* mbuf *);
121	static struct socket sonewconn_internal(struct* socket , int*);
122	static int sbappendcontrol_internal(struct sockbuf , struct* mbuf *,
123	struct mbuf *);
124	static void soevent_ifdenied(struct socket *);
125
126	static int sbappendrecord_common(struct sockbuf sb, struct* mbuf *m0, boolean_t nodrop);
127	static int sbappend_common(struct sockbuf sb, struct* mbuf *m, boolean_t nodrop);
128
129	/*
130	* Primitive routines for operating on sockets and socket buffers
131	*/
132	static int soqlimitcompat = `1`;
133	static int soqlencomp = `0`;
134
135	/*
136	* Based on the number of mbuf clusters configured, high_sb_max and sb_max can
137	* get scaled up or down to suit that memory configuration. high_sb_max is a
138	* higher limit on sb_max that is checked when sb_max gets set through sysctl.
139	*/
140	uint32_t sb_max = SB_MAX;
141	uint32_t high_sb_max = SB_MAX;
142
143	static uint32_t sb_efficiency = `8`; / parameter for sbreserve() /
144
145	uint32_t net_io_policy_log = `0`; / log socket policy changes /
146	#if CONFIG_PROC_UUID_POLICY
147	uint32_t net_io_policy_uuid = `1`; / enable UUID socket policy /
148	#endif /* CONFIG_PROC_UUID_POLICY */
149
150	/*
151	* Procedures to manipulate state flags of socket
152	* and do appropriate wakeups. Normal sequence from the
153	* active (originating) side is that soisconnecting() is
154	* called during processing of connect() call,
155	* resulting in an eventual call to soisconnected() if/when the
156	* connection is established. When the connection is torn down
157	* soisdisconnecting() is called during processing of disconnect() call,
158	* and soisdisconnected() is called when the connection to the peer
159	* is totally severed. The semantics of these routines are such that
160	* connectionless protocols can call soisconnected() and soisdisconnected()
161	* only, bypassing the in-progress calls when setting up a ``connection''
162	* takes no time.
163	*
164	* From the passive side, a socket is created with
165	* two queues of sockets: so_incomp for connections in progress
166	* and so_comp for connections already made and awaiting user acceptance.
167	* As a protocol is preparing incoming connections, it creates a socket
168	* structure queued on so_incomp by calling sonewconn(). When the connection
169	* is established, soisconnected() is called, and transfers the
170	* socket structure to so_comp, making it available to accept().
171	*
172	* If a socket is closed with sockets on either
173	* so_incomp or so_comp, these sockets are dropped.
174	*
175	* If higher level protocols are implemented in
176	* the kernel, the wakeups done here will sometimes
177	* cause software-interrupt process scheduling.
178	*/
179	void
180	soisconnecting(struct socket *so)
181	{
182	so->so_state &= ~(SS_ISCONNECTED \| SS_ISDISCONNECTING);
183	so->so_state \|= SS_ISCONNECTING;
184
185	sflt_notify(so, event: sock_evt_connecting, NULL);
186	}
187
188	void
189	soisconnected(struct socket *so)
190	{
191	/*
192	* If socket is subject to filter and is pending initial verdict,
193	* delay marking socket as connected and do not present the connected
194	* socket to user just yet.
195	*/
196	if (cfil_sock_connected_pending_verdict(so)) {
197	return;
198	}
199
200	so->so_state &= ~(SS_ISCONNECTING \| SS_ISDISCONNECTING \| SS_ISCONFIRMING);
201	so->so_state \|= SS_ISCONNECTED;
202
203	soreserve_preconnect(so, pre_cc: `0`);
204
205	sflt_notify(so, event: sock_evt_connected, NULL);
206
207	if (so->so_head != NULL && (so->so_state & SS_INCOMP)) {
208	struct socket *head = so->so_head;
209	int locked = `0`;
210
211	/*
212	* Enforce lock order when the protocol has per socket locks
213	*/
214	if (head->so_proto->pr_getlock != NULL) {
215	socket_lock(so: head, refcount: `1`);
216	so_acquire_accept_list(head, so);
217	locked = `1`;
218	}
219	if (so->so_head == head && (so->so_state & SS_INCOMP)) {
220	so->so_state &= ~SS_INCOMP;
221	so->so_state \|= SS_COMP;
222	TAILQ_REMOVE(&head->so_incomp, so, so_list);
223	TAILQ_INSERT_TAIL(&head->so_comp, so, so_list);
224	head->so_incqlen--;
225
226	/*
227	* We have to release the accept list in
228	* case a socket callback calls sock_accept()
229	*/
230	if (locked != `0`) {
231	so_release_accept_list(head);
232	socket_unlock(so, refcount: `0`);
233	}
234	sorwakeup(so: head);
235	wakeup_one(chan: (caddr_t)&head->so_timeo);
236
237	if (locked != `0`) {
238	socket_unlock(so: head, refcount: `1`);
239	socket_lock(so, refcount: `0`);
240	}
241	} else if (locked != `0`) {
242	so_release_accept_list(head);
243	socket_unlock(so: head, refcount: `1`);
244	}
245	} else {
246	wakeup(chan: (caddr_t)&so->so_timeo);
247	sorwakeup(so);
248	sowwakeup(so);
249	soevent(so, SO_FILT_HINT_LOCKED \| SO_FILT_HINT_CONNECTED \|
250	SO_FILT_HINT_CONNINFO_UPDATED);
251	}
252	}
253
254	boolean_t
255	socanwrite(struct socket *so)
256	{
257	return (so->so_state & SS_ISCONNECTED) \|\|
258	!(so->so_proto->pr_flags & PR_CONNREQUIRED) \|\|
259	(so->so_flags1 & SOF1_PRECONNECT_DATA);
260	}
261
262	void
263	soisdisconnecting(struct socket *so)
264	{
265	so->so_state &= ~SS_ISCONNECTING;
266	so->so_state \|= (SS_ISDISCONNECTING \| SS_CANTRCVMORE \| SS_CANTSENDMORE);
267	soevent(so, SO_FILT_HINT_LOCKED);
268	sflt_notify(so, event: sock_evt_disconnecting, NULL);
269	wakeup(chan: (caddr_t)&so->so_timeo);
270	sowwakeup(so);
271	sorwakeup(so);
272	}
273
274	void
275	soisdisconnected(struct socket *so)
276	{
277	so->so_state &= ~(SS_ISCONNECTING \| SS_ISCONNECTED \| SS_ISDISCONNECTING);
278	so->so_state \|= (SS_CANTRCVMORE \| SS_CANTSENDMORE \| SS_ISDISCONNECTED);
279	soevent(so, SO_FILT_HINT_LOCKED \| SO_FILT_HINT_DISCONNECTED \|
280	SO_FILT_HINT_CONNINFO_UPDATED);
281	sflt_notify(so, event: sock_evt_disconnected, NULL);
282	wakeup(chan: (caddr_t)&so->so_timeo);
283	sowwakeup(so);
284	sorwakeup(so);
285
286	#if CONTENT_FILTER
287	/ Notify content filters as soon as we cannot send/receive data /
288	cfil_sock_notify_shutdown(so, SHUT_RDWR);
289	#endif /* CONTENT_FILTER */
290	}
291
292	/*
293	* This function will issue a wakeup like soisdisconnected but it will not
294	* notify the socket filters. This will avoid unlocking the socket
295	* in the midst of closing it.
296	*/
297	void
298	sodisconnectwakeup(struct socket *so)
299	{
300	so->so_state &= ~(SS_ISCONNECTING \| SS_ISCONNECTED \| SS_ISDISCONNECTING);
301	so->so_state \|= (SS_CANTRCVMORE \| SS_CANTSENDMORE \| SS_ISDISCONNECTED);
302	soevent(so, SO_FILT_HINT_LOCKED \| SO_FILT_HINT_DISCONNECTED \|
303	SO_FILT_HINT_CONNINFO_UPDATED);
304	wakeup(chan: (caddr_t)&so->so_timeo);
305	sowwakeup(so);
306	sorwakeup(so);
307
308	#if CONTENT_FILTER
309	/ Notify content filters as soon as we cannot send/receive data /
310	cfil_sock_notify_shutdown(so, SHUT_RDWR);
311	#endif /* CONTENT_FILTER */
312	}
313
314	/*
315	* When an attempt at a new connection is noted on a socket
316	* which accepts connections, sonewconn is called. If the
317	* connection is possible (subject to space constraints, etc.)
318	* then we allocate a new structure, propoerly linked into the
319	* data structure of the original socket, and return this.
320	* Connstatus may be 0, or SO_ISCONFIRMING, or SO_ISCONNECTED.
321	*/
322	static struct socket *
323	sonewconn_internal(struct socket head, int* connstatus)
324	{
325	int so_qlen, error = `0`;
326	struct socket *so;
327	lck_mtx_t *mutex_held;
328
329	if (head->so_proto->pr_getlock != NULL) {
330	mutex_held = (*head->so_proto->pr_getlock)(head, `0`);
331	} else {
332	mutex_held = head->so_proto->pr_domain->dom_mtx;
333	}
334	LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
335
336	if (!soqlencomp) {
337	/*
338	* This is the default case; so_qlen represents the
339	* sum of both incomplete and completed queues.
340	*/
341	so_qlen = head->so_qlen;
342	} else {
343	/*
344	* When kern.ipc.soqlencomp is set to 1, so_qlen
345	* represents only the completed queue. Since we
346	* cannot let the incomplete queue goes unbounded
347	* (in case of SYN flood), we cap the incomplete
348	* queue length to at most somaxconn, and use that
349	* as so_qlen so that we fail immediately below.
350	*/
351	so_qlen = head->so_qlen - head->so_incqlen;
352	if (head->so_incqlen > somaxconn) {
353	so_qlen = somaxconn;
354	}
355	}
356
357	if (so_qlen >=
358	(soqlimitcompat ? head->so_qlimit : (`3` * head->so_qlimit / `2`))) {
359	return (struct socket *)`0`;
360	}
361	so = soalloc(waitok: `1`, SOCK_DOM(head), type: head->so_type);
362	if (so == NULL) {
363	return (struct socket *)`0`;
364	}
365	/ check if head was closed during the soalloc /
366	if (head->so_proto == NULL) {
367	sodealloc(so);
368	return (struct socket *)`0`;
369	}
370
371	so->so_type = head->so_type;
372	so->so_family = head->so_family;
373	so->so_protocol = head->so_protocol;
374	so->so_options = head->so_options & ~SO_ACCEPTCONN;
375	so->so_linger = head->so_linger;
376	so->so_state = head->so_state \| SS_NOFDREF;
377	so->so_proto = head->so_proto;
378	so->so_timeo = head->so_timeo;
379	so->so_pgid = head->so_pgid;
380	kauth_cred_ref(cred: head->so_cred);
381	so->so_cred = head->so_cred;
382	so->so_persona_id = head->so_persona_id;
383	so->last_pid = head->last_pid;
384	so->last_upid = head->last_upid;
385	memcpy(dst: so->last_uuid, src: head->last_uuid, n: sizeof(so->last_uuid));
386	if (head->so_flags & SOF_DELEGATED) {
387	so->e_pid = head->e_pid;
388	so->e_upid = head->e_upid;
389	memcpy(dst: so->e_uuid, src: head->e_uuid, n: sizeof(so->e_uuid));
390	}
391	/ inherit socket options stored in so_flags /
392	so->so_flags = head->so_flags &
393	(SOF_NOSIGPIPE \| SOF_NOADDRAVAIL \| SOF_REUSESHAREUID \|
394	SOF_NOTIFYCONFLICT \| SOF_BINDRANDOMPORT \| SOF_NPX_SETOPTSHUT \|
395	SOF_NODEFUNCT \| SOF_PRIVILEGED_TRAFFIC_CLASS \| SOF_NOTSENT_LOWAT \|
396	SOF_DELEGATED);
397	so->so_flags1 \|= SOF1_INBOUND;
398	so->so_usecount = `1`;
399	so->next_lock_lr = `0`;
400	so->next_unlock_lr = `0`;
401
402	so->so_rcv.sb_flags \|= SB_RECV; / XXX /
403	so->so_rcv.sb_so = so->so_snd.sb_so = so;
404
405	/ inherit traffic management properties of listener /
406	so->so_flags1 \|=
407	head->so_flags1 & (SOF1_TRAFFIC_MGT_SO_BACKGROUND \| SOF1_TC_NET_SERV_TYPE \|
408	SOF1_QOSMARKING_ALLOWED \| SOF1_QOSMARKING_POLICY_OVERRIDE);
409	so->so_background_thread = head->so_background_thread;
410	so->so_traffic_class = head->so_traffic_class;
411	so->so_netsvctype = head->so_netsvctype;
412
413	if (soreserve(so, sndcc: head->so_snd.sb_hiwat, rcvcc: head->so_rcv.sb_hiwat)) {
414	sodealloc(so);
415	return (struct socket *)`0`;
416	}
417	so->so_rcv.sb_flags \|= (head->so_rcv.sb_flags & SB_USRSIZE);
418	so->so_snd.sb_flags \|= (head->so_snd.sb_flags & SB_USRSIZE);
419
420	/*
421	* Must be done with head unlocked to avoid deadlock
422	* for protocol with per socket mutexes.
423	*/
424	if (head->so_proto->pr_unlock) {
425	socket_unlock(so: head, refcount: `0`);
426	}
427	if (((*so->so_proto->pr_usrreqs->pru_attach)(so, `0`, NULL) != `0`) \|\|
428	error) {
429	sodealloc(so);
430	if (head->so_proto->pr_unlock) {
431	socket_lock(so: head, refcount: `0`);
432	}
433	return (struct socket *)`0`;
434	}
435	if (head->so_proto->pr_unlock) {
436	socket_lock(so: head, refcount: `0`);
437	/*
438	* Radar 7385998 Recheck that the head is still accepting
439	* to avoid race condition when head is getting closed.
440	*/
441	if ((head->so_options & SO_ACCEPTCONN) == `0`) {
442	so->so_state &= ~SS_NOFDREF;
443	soclose(so);
444	return (struct socket *)`0`;
445	}
446	}
447
448	if (so->so_proto->pr_copy_last_owner != NULL) {
449	(*so->so_proto->pr_copy_last_owner)(so, head);
450	}
451	os_atomic_inc(&so->so_proto->pr_domain->dom_refs, relaxed);
452
453	/ Insert in head appropriate lists /
454	so_acquire_accept_list(head, NULL);
455
456	so->so_head = head;
457
458	/*
459	* Since this socket is going to be inserted into the incomp
460	* queue, it can be picked up by another thread in
461	* tcp_dropdropablreq to get dropped before it is setup..
462	* To prevent this race, set in-progress flag which can be
463	* cleared later
464	*/
465	so->so_flags \|= SOF_INCOMP_INPROGRESS;
466
467	if (connstatus) {
468	TAILQ_INSERT_TAIL(&head->so_comp, so, so_list);
469	so->so_state \|= SS_COMP;
470	} else {
471	TAILQ_INSERT_TAIL(&head->so_incomp, so, so_list);
472	so->so_state \|= SS_INCOMP;
473	head->so_incqlen++;
474	}
475	head->so_qlen++;
476
477	so_release_accept_list(head);
478
479	/ Attach socket filters for this protocol /
480	sflt_initsock(so);
481
482	if (connstatus) {
483	so->so_state \|= (short)connstatus;
484	sorwakeup(so: head);
485	wakeup(chan: (caddr_t)&head->so_timeo);
486	}
487	return so;
488	}
489
490
491	struct socket *
492	sonewconn(struct socket head, int* connstatus, const struct sockaddr *from)
493	{
494	int error = sflt_connectin(head, remote: from);
495	if (error) {
496	return NULL;
497	}
498
499	return sonewconn_internal(head, connstatus);
500	}
501
502	/*
503	* Socantsendmore indicates that no more data will be sent on the
504	* socket; it would normally be applied to a socket when the user
505	* informs the system that no more data is to be sent, by the protocol
506	* code (in case PRU_SHUTDOWN). Socantrcvmore indicates that no more data
507	* will be received, and will normally be applied to the socket by a
508	* protocol when it detects that the peer will send no more data.
509	* Data queued for reading in the socket may yet be read.
510	*/
511
512	void
513	socantsendmore(struct socket *so)
514	{
515	so->so_state \|= SS_CANTSENDMORE;
516	soevent(so, SO_FILT_HINT_LOCKED \| SO_FILT_HINT_CANTSENDMORE);
517	sflt_notify(so, event: sock_evt_cantsendmore, NULL);
518	sowwakeup(so);
519	}
520
521	void
522	socantrcvmore(struct socket *so)
523	{
524	so->so_state \|= SS_CANTRCVMORE;
525	soevent(so, SO_FILT_HINT_LOCKED \| SO_FILT_HINT_CANTRCVMORE);
526	sflt_notify(so, event: sock_evt_cantrecvmore, NULL);
527	sorwakeup(so);
528	}
529
530	/*
531	* Wait for data to arrive at/drain from a socket buffer.
532	*/
533	int
534	sbwait(struct sockbuf *sb)
535	{
536	boolean_t nointr = (sb->sb_flags & SB_NOINTR);
537	void *lr_saved = __builtin_return_address(`0`);
538	struct socket *so = sb->sb_so;
539	lck_mtx_t *mutex_held;
540	struct timespec ts;
541	int error = `0`;
542
543	if (so == NULL) {
544	panic("%s: null so, sb=%p sb_flags=0x%x lr=%p",
545	__func__, sb, sb->sb_flags, lr_saved);
546	/ NOTREACHED /
547	} else if (so->so_usecount < `1`) {
548	panic("%s: sb=%p sb_flags=0x%x sb_so=%p usecount=%d lr=%p "
549	"lrh= %s\n", __func__, sb, sb->sb_flags, so,
550	so->so_usecount, lr_saved, solockhistory_nr(so));
551	/ NOTREACHED /
552	}
553
554	if ((so->so_state & SS_DRAINING) \|\| (so->so_flags & SOF_DEFUNCT)) {
555	error = EBADF;
556	if (so->so_flags & SOF_DEFUNCT) {
557	SODEFUNCTLOG("%s[%d, %s]: defunct so 0x%llu [%d,%d] "
558	"(%d)\n", __func__, proc_selfpid(),
559	proc_best_name(current_proc()),
560	so->so_gencnt,
561	SOCK_DOM(so), SOCK_TYPE(so), error);
562	}
563	return error;
564	}
565
566	if (so->so_proto->pr_getlock != NULL) {
567	mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK);
568	} else {
569	mutex_held = so->so_proto->pr_domain->dom_mtx;
570	}
571
572	LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
573
574	ts.tv_sec = sb->sb_timeo.tv_sec;
575	ts.tv_nsec = sb->sb_timeo.tv_usec * `1000`;
576
577	sb->sb_waiters++;
578	VERIFY(sb->sb_waiters != `0`);
579
580	error = msleep(chan: (caddr_t)&sb->sb_cc, mtx: mutex_held,
581	pri: nointr ? PSOCK : PSOCK \| PCATCH,
582	wmesg: nointr ? "sbwait_nointr" : "sbwait", ts: &ts);
583
584	VERIFY(sb->sb_waiters != `0`);
585	sb->sb_waiters--;
586
587	if (so->so_usecount < `1`) {
588	panic("%s: 2 sb=%p sb_flags=0x%x sb_so=%p usecount=%d lr=%p "
589	"lrh= %s\n", __func__, sb, sb->sb_flags, so,
590	so->so_usecount, lr_saved, solockhistory_nr(so));
591	/ NOTREACHED /
592	}
593
594	if ((so->so_state & SS_DRAINING) \|\| (so->so_flags & SOF_DEFUNCT)) {
595	error = EBADF;
596	if (so->so_flags & SOF_DEFUNCT) {
597	SODEFUNCTLOG("%s[%d, %s]: defunct so 0x%llu [%d,%d] "
598	"(%d)\n", __func__, proc_selfpid(),
599	proc_best_name(current_proc()),
600	so->so_gencnt,
601	SOCK_DOM(so), SOCK_TYPE(so), error);
602	}
603	}
604
605	return error;
606	}
607
608	void
609	sbwakeup(struct sockbuf *sb)
610	{
611	if (sb->sb_waiters > `0`) {
612	wakeup(chan: (caddr_t)&sb->sb_cc);
613	}
614	}
615
616	/*
617	* Wakeup processes waiting on a socket buffer.
618	* Do asynchronous notification via SIGIO
619	* if the socket has the SS_ASYNC flag set.
620	*/
621	void
622	sowakeup(struct socket so, struct* sockbuf sb, struct* socket *so2)
623	{
624	if (so->so_flags & SOF_DEFUNCT) {
625	SODEFUNCTLOG("%s[%d, %s]: defunct so 0x%llu [%d,%d] si 0x%x, "
626	"fl 0x%x [%s]\n", __func__, proc_selfpid(),
627	proc_best_name(current_proc()),
628	so->so_gencnt, SOCK_DOM(so),
629	SOCK_TYPE(so), (uint32_t)sb->sb_sel.si_flags, sb->sb_flags,
630	(sb->sb_flags & SB_RECV) ? "rcv" : "snd");
631	}
632
633	sb->sb_flags &= ~SB_SEL;
634	selwakeup(&sb->sb_sel);
635	sbwakeup(sb);
636	if (so->so_state & SS_ASYNC) {
637	if (so->so_pgid < `0`) {
638	gsignal(pgid: -so->so_pgid, SIGIO);
639	} else if (so->so_pgid > `0`) {
640	proc_signal(pid: so->so_pgid, SIGIO);
641	}
642	}
643	if (sb->sb_flags & SB_KNOTE) {
644	KNOTE(&sb->sb_sel.si_note, SO_FILT_HINT_LOCKED);
645	}
646	if (sb->sb_flags & SB_UPCALL) {
647	void (sb_upcall)(struct* socket , void* , int*);
648	caddr_t sb_upcallarg;
649	int lock = !(sb->sb_flags & SB_UPCALL_LOCK);
650
651	sb_upcall = sb->sb_upcall;
652	sb_upcallarg = sb->sb_upcallarg;
653	/ Let close know that we're about to do an upcall /
654	so->so_upcallusecount++;
655
656	if (lock) {
657	if (so2) {
658	struct unpcb *unp = sotounpcb(so2);
659	unp->unp_flags \|= UNP_DONTDISCONNECT;
660	unp->rw_thrcount++;
661
662	socket_unlock(so: so2, refcount: `0`);
663	}
664	socket_unlock(so, refcount: `0`);
665	}
666	(*sb_upcall)(so, sb_upcallarg, M_DONTWAIT);
667	if (lock) {
668	if (so2 && so > so2) {
669	struct unpcb *unp;
670	socket_lock(so: so2, refcount: `0`);
671
672	unp = sotounpcb(so2);
673	unp->rw_thrcount--;
674	if (unp->rw_thrcount == `0`) {
675	unp->unp_flags &= ~UNP_DONTDISCONNECT;
676	wakeup(chan: unp);
677	}
678	}
679
680	socket_lock(so, refcount: `0`);
681
682	if (so2 && so < so2) {
683	struct unpcb *unp;
684	socket_lock(so: so2, refcount: `0`);
685
686	unp = sotounpcb(so2);
687	unp->rw_thrcount--;
688	if (unp->rw_thrcount == `0`) {
689	unp->unp_flags &= ~UNP_DONTDISCONNECT;
690	wakeup(chan: unp);
691	}
692	}
693	}
694
695	so->so_upcallusecount--;
696	/ Tell close that it's safe to proceed /
697	if ((so->so_flags & SOF_CLOSEWAIT) &&
698	so->so_upcallusecount == `0`) {
699	wakeup(chan: (caddr_t)&so->so_upcallusecount);
700	}
701	}
702	#if CONTENT_FILTER
703	/*
704	* Trap disconnection events for content filters
705	*/
706	if ((so->so_flags & SOF_CONTENT_FILTER) != `0`) {
707	if ((sb->sb_flags & SB_RECV)) {
708	if (so->so_state & (SS_CANTRCVMORE)) {
709	cfil_sock_notify_shutdown(so, SHUT_RD);
710	}
711	} else {
712	if (so->so_state & (SS_CANTSENDMORE)) {
713	cfil_sock_notify_shutdown(so, SHUT_WR);
714	}
715	}
716	}
717	#endif /* CONTENT_FILTER */
718	}
719
720	/*
721	* Socket buffer (struct sockbuf) utility routines.
722	*
723	* Each socket contains two socket buffers: one for sending data and
724	* one for receiving data. Each buffer contains a queue of mbufs,
725	* information about the number of mbufs and amount of data in the
726	* queue, and other fields allowing select() statements and notification
727	* on data availability to be implemented.
728	*
729	* Data stored in a socket buffer is maintained as a list of records.
730	* Each record is a list of mbufs chained together with the m_next
731	* field. Records are chained together with the m_nextpkt field. The upper
732	* level routine soreceive() expects the following conventions to be
733	* observed when placing information in the receive buffer:
734	*
735	* 1. If the protocol requires each message be preceded by the sender's
736	* name, then a record containing that name must be present before
737	* any associated data (mbuf's must be of type MT_SONAME).
738	* 2. If the protocol supports the exchange of ``access rights'' (really
739	* just additional data associated with the message), and there are
740	* ``rights'' to be received, then a record containing this data
741	* should be present (mbuf's must be of type MT_RIGHTS).
742	* 3. If a name or rights record exists, then it must be followed by
743	* a data record, perhaps of zero length.
744	*
745	* Before using a new socket structure it is first necessary to reserve
746	* buffer space to the socket, by calling sbreserve(). This should commit
747	* some of the available buffer space in the system buffer pool for the
748	* socket (currently, it does nothing but enforce limits). The space
749	* should be released by calling sbrelease() when the socket is destroyed.
750	*/
751
752	/*
753	* Returns: 0 Success
754	* ENOBUFS
755	*/
756	int
757	soreserve(struct socket *so, uint32_t sndcc, uint32_t rcvcc)
758	{
759	if (sbreserve(sb: &so->so_snd, cc: sndcc) == `0`) {
760	goto bad;
761	} else {
762	so->so_snd.sb_idealsize = sndcc;
763	}
764
765	if (sbreserve(sb: &so->so_rcv, cc: rcvcc) == `0`) {
766	goto bad2;
767	} else {
768	so->so_rcv.sb_idealsize = rcvcc;
769	}
770
771	if (so->so_rcv.sb_lowat == `0`) {
772	so->so_rcv.sb_lowat = `1`;
773	}
774	if (so->so_snd.sb_lowat == `0`) {
775	so->so_snd.sb_lowat = MCLBYTES;
776	}
777	if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) {
778	so->so_snd.sb_lowat = so->so_snd.sb_hiwat;
779	}
780	return `0`;
781	bad2:
782	so->so_snd.sb_flags &= ~SB_SEL;
783	selthreadclear(&so->so_snd.sb_sel);
784	sbrelease(sb: &so->so_snd);
785	bad:
786	return ENOBUFS;
787	}
788
789	void
790	soreserve_preconnect(struct socket so, unsigned* int pre_cc)
791	{
792	/ As of now, same bytes for both preconnect read and write /
793	so->so_snd.sb_preconn_hiwat = pre_cc;
794	so->so_rcv.sb_preconn_hiwat = pre_cc;
795	}
796
797	/*
798	* Allot mbufs to a sockbuf.
799	* Attempt to scale mbmax so that mbcnt doesn't become limiting
800	* if buffering efficiency is near the normal case.
801	*/
802	int
803	sbreserve(struct sockbuf *sb, uint32_t cc)
804	{
805	if (cc > sb_max) {
806	/ We would not end up changing sb_cc, so return 0 /
807	if (sb->sb_hiwat == sb_max) {
808	return `0`;
809	}
810	cc = sb_max;
811	}
812	if (cc > sb->sb_hiwat && (sb->sb_flags & SB_LIMITED)) {
813	return `0`;
814	}
815	sb->sb_hiwat = cc;
816	sb->sb_mbmax = cc * sb_efficiency;
817	if (sb->sb_lowat > sb->sb_hiwat) {
818	sb->sb_lowat = sb->sb_hiwat;
819	}
820	return `1`;
821	}
822
823	/*
824	* Free mbufs held by a socket, and reserved mbuf space.
825	*/
826	/ WARNING needs to do selthreadclear() before calling this /
827	void
828	sbrelease(struct sockbuf *sb)
829	{
830	sbflush(sb);
831	sb->sb_hiwat = `0`;
832	sb->sb_mbmax = `0`;
833	}
834
835	/*
836	* Routines to add and remove
837	* data from an mbuf queue.
838	*
839	* The routines sbappend() or sbappendrecord() are normally called to
840	* append new mbufs to a socket buffer, after checking that adequate
841	* space is available, comparing the function sbspace() with the amount
842	* of data to be added. sbappendrecord() differs from sbappend() in
843	* that data supplied is treated as the beginning of a new record.
844	* To place a sender's address, optional access rights, and data in a
845	* socket receive buffer, sbappendaddr() should be used. To place
846	* access rights and data in a socket receive buffer, sbappendrights()
847	* should be used. In either case, the new data begins a new record.
848	* Note that unlike sbappend() and sbappendrecord(), these routines check
849	* for the caller that there will be enough space to store the data.
850	* Each fails if there is not enough space, or if it cannot find mbufs
851	* to store additional information in.
852	*
853	* Reliable protocols may use the socket send buffer to hold data
854	* awaiting acknowledgement. Data is normally copied from a socket
855	* send buffer in a protocol with m_copy for output to a peer,
856	* and then removing the data from the socket buffer with sbdrop()
857	* or sbdroprecord() when the data is acknowledged by the peer.
858	*/
859
860	/*
861	* Append mbuf chain m to the last record in the
862	* socket buffer sb. The additional space associated
863	* the mbuf chain is recorded in sb. Empty mbufs are
864	* discarded and mbufs are compacted where possible.
865	*/
866	static int
867	sbappend_common(struct sockbuf sb, struct* mbuf *m, boolean_t nodrop)
868	{
869	struct socket *so = sb->sb_so;
870	struct soflow_hash_entry *dgram_flow_entry = NULL;
871
872	if (m == NULL \|\| (sb->sb_flags & SB_DROP)) {
873	if (m != NULL && !nodrop) {
874	m_freem(m);
875	}
876	return `0`;
877	}
878
879	SBLASTRECORDCHK(sb, "sbappend 1");
880
881	if (sb->sb_lastrecord != NULL && (sb->sb_mbtail->m_flags & M_EOR)) {
882	return sbappendrecord_common(sb, m0: m, nodrop);
883	}
884
885	if (SOCK_DOM(sb->sb_so) == PF_INET \|\| SOCK_DOM(sb->sb_so) == PF_INET6) {
886	ASSERT(nodrop == FALSE);
887
888	if (NEED_DGRAM_FLOW_TRACKING(so)) {
889	dgram_flow_entry = soflow_get_flow(so, NULL, NULL, NULL, m != NULL ? m_length(m) : `0`, false, (m != NULL && m->m_pkthdr.rcvif) ? m->m_pkthdr.rcvif->if_index : `0`);
890	}
891
892	if (sb->sb_flags & SB_RECV && !(m && m->m_flags & M_SKIPCFIL)) {
893	int error = sflt_data_in(so, NULL, data: &m, NULL, flags: `0`);
894	SBLASTRECORDCHK(sb, "sbappend 2");
895
896	#if CONTENT_FILTER
897	if (error == `0`) {
898	error = cfil_sock_data_in(so, NULL, data: m, NULL, flags: `0`, dgram_flow_entry);
899	}
900	#endif /* CONTENT_FILTER */
901
902	if (error != `0`) {
903	if (error != EJUSTRETURN) {
904	m_freem(m);
905	}
906	if (dgram_flow_entry != NULL) {
907	soflow_free_flow(dgram_flow_entry);
908	}
909	return `0`;
910	}
911	} else if (m) {
912	m->m_flags &= ~M_SKIPCFIL;
913	}
914
915	if (dgram_flow_entry != NULL) {
916	soflow_free_flow(dgram_flow_entry);
917	}
918	}
919
920	/ If this is the first record, it's also the last record /
921	if (sb->sb_lastrecord == NULL) {
922	sb->sb_lastrecord = m;
923	}
924
925	sbcompress(sb, m, sb->sb_mbtail);
926	SBLASTRECORDCHK(sb, "sbappend 3");
927	return `1`;
928	}
929
930	int
931	sbappend(struct sockbuf sb, struct* mbuf *m)
932	{
933	return sbappend_common(sb, m, FALSE);
934	}
935
936	int
937	sbappend_nodrop(struct sockbuf sb, struct* mbuf *m)
938	{
939	return sbappend_common(sb, m, TRUE);
940	}
941
942	/*
943	* Similar to sbappend, except that this is optimized for stream sockets.
944	*/
945	int
946	sbappendstream(struct sockbuf sb, struct* mbuf *m)
947	{
948	struct soflow_hash_entry *dgram_flow_entry = NULL;
949	struct socket *so = sb->sb_so;
950
951	if (m == NULL \|\| (sb->sb_flags & SB_DROP)) {
952	if (m != NULL) {
953	m_freem(m);
954	}
955	return `0`;
956	}
957
958	if (m->m_nextpkt != NULL \|\| (sb->sb_mb != sb->sb_lastrecord)) {
959	panic("sbappendstream: nexpkt %p \|\| mb %p != lastrecord %p",
960	m->m_nextpkt, sb->sb_mb, sb->sb_lastrecord);
961	/ NOTREACHED /
962	}
963
964	SBLASTMBUFCHK(sb, __func__);
965
966	if (SOCK_DOM(sb->sb_so) == PF_INET \|\| SOCK_DOM(sb->sb_so) == PF_INET6) {
967	if (NEED_DGRAM_FLOW_TRACKING(so)) {
968	dgram_flow_entry = soflow_get_flow(so, NULL, NULL, NULL, m != NULL ? m_length(m) : `0`, false, (m != NULL && m->m_pkthdr.rcvif) ? m->m_pkthdr.rcvif->if_index : `0`);
969	}
970
971	if (sb->sb_flags & SB_RECV && !(m && m->m_flags & M_SKIPCFIL)) {
972	int error = sflt_data_in(so, NULL, data: &m, NULL, flags: `0`);
973	SBLASTRECORDCHK(sb, "sbappendstream 1");
974
975	#if CONTENT_FILTER
976	if (error == `0`) {
977	error = cfil_sock_data_in(so, NULL, data: m, NULL, flags: `0`, dgram_flow_entry);
978	}
979	#endif /* CONTENT_FILTER */
980
981	if (error != `0`) {
982	if (error != EJUSTRETURN) {
983	m_freem(m);
984	}
985	if (dgram_flow_entry != NULL) {
986	soflow_free_flow(dgram_flow_entry);
987	}
988	return `0`;
989	}
990	} else if (m) {
991	m->m_flags &= ~M_SKIPCFIL;
992	}
993
994	if (dgram_flow_entry != NULL) {
995	soflow_free_flow(dgram_flow_entry);
996	}
997	}
998
999	sbcompress(sb, m, sb->sb_mbtail);
1000	sb->sb_lastrecord = sb->sb_mb;
1001	SBLASTRECORDCHK(sb, "sbappendstream 2");
1002	return `1`;
1003	}
1004
1005	#ifdef SOCKBUF_DEBUG
1006	void
1007	sbcheck(struct sockbuf *sb)
1008	{
1009	struct mbuf *m;
1010	struct mbuf *n = `0`;
1011	u_int32_t len = `0`, mbcnt = `0`;
1012	lck_mtx_t *mutex_held;
1013
1014	if (sb->sb_so->so_proto->pr_getlock != NULL) {
1015	mutex_held = (*sb->sb_so->so_proto->pr_getlock)(sb->sb_so, `0`);
1016	} else {
1017	mutex_held = sb->sb_so->so_proto->pr_domain->dom_mtx;
1018	}
1019
1020	LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
1021
1022	if (sbchecking == `0`) {
1023	return;
1024	}
1025
1026	for (m = sb->sb_mb; m; m = n) {
1027	n = m->m_nextpkt;
1028	for (; m; m = m->m_next) {
1029	len += m->m_len;
1030	mbcnt += _MSIZE;
1031	/ XXX pretty sure this is bogus /
1032	if (m->m_flags & M_EXT) {
1033	mbcnt += m->m_ext.ext_size;
1034	}
1035	}
1036	}
1037	if (len != sb->sb_cc \|\| mbcnt != sb->sb_mbcnt) {
1038	panic("cc %ld != %ld \|\| mbcnt %ld != %ld", len, sb->sb_cc,
1039	mbcnt, sb->sb_mbcnt);
1040	}
1041	}
1042	#endif
1043
1044	void
1045	sblastrecordchk(struct sockbuf sb, const* char *where)
1046	{
1047	struct mbuf *m = sb->sb_mb;
1048
1049	while (m && m->m_nextpkt) {
1050	m = m->m_nextpkt;
1051	}
1052
1053	if (m != sb->sb_lastrecord) {
1054	printf("sblastrecordchk: mb 0x%llx lastrecord 0x%llx "
1055	"last 0x%llx\n",
1056	(uint64_t)VM_KERNEL_ADDRPERM(sb->sb_mb),
1057	(uint64_t)VM_KERNEL_ADDRPERM(sb->sb_lastrecord),
1058	(uint64_t)VM_KERNEL_ADDRPERM(m));
1059	printf("packet chain:\n");
1060	for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) {
1061	printf("\t0x%llx\n", (uint64_t)VM_KERNEL_ADDRPERM(m));
1062	}
1063	panic("sblastrecordchk from %s", where);
1064	}
1065	}
1066
1067	void
1068	sblastmbufchk(struct sockbuf sb, const* char *where)
1069	{
1070	struct mbuf *m = sb->sb_mb;
1071	struct mbuf *n;
1072
1073	while (m && m->m_nextpkt) {
1074	m = m->m_nextpkt;
1075	}
1076
1077	while (m && m->m_next) {
1078	m = m->m_next;
1079	}
1080
1081	if (m != sb->sb_mbtail) {
1082	printf("sblastmbufchk: mb 0x%llx mbtail 0x%llx last 0x%llx\n",
1083	(uint64_t)VM_KERNEL_ADDRPERM(sb->sb_mb),
1084	(uint64_t)VM_KERNEL_ADDRPERM(sb->sb_mbtail),
1085	(uint64_t)VM_KERNEL_ADDRPERM(m));
1086	printf("packet tree:\n");
1087	for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) {
1088	printf("\t");
1089	for (n = m; n != NULL; n = n->m_next) {
1090	printf("0x%llx ",
1091	(uint64_t)VM_KERNEL_ADDRPERM(n));
1092	}
1093	printf("\n");
1094	}
1095	panic("sblastmbufchk from %s", where);
1096	}
1097	}
1098
1099	/*
1100	* Similar to sbappend, except the mbuf chain begins a new record.
1101	*/
1102	static int
1103	sbappendrecord_common(struct sockbuf sb, struct* mbuf *m0, boolean_t nodrop)
1104	{
1105	struct soflow_hash_entry *dgram_flow_entry = NULL;
1106	struct socket *so = sb->sb_so;
1107	struct mbuf *m;
1108	int space = `0`;
1109
1110	if (m0 == NULL \|\| (sb->sb_flags & SB_DROP)) {
1111	if (m0 != NULL && nodrop == FALSE) {
1112	m_freem(m0);
1113	}
1114	return `0`;
1115	}
1116
1117	for (m = m0; m != NULL; m = m->m_next) {
1118	space += m->m_len;
1119	}
1120
1121	if (space > sbspace(sb) && !(sb->sb_flags & SB_UNIX)) {
1122	if (nodrop == FALSE) {
1123	m_freem(m0);
1124	}
1125	return `0`;
1126	}
1127
1128	if (SOCK_DOM(sb->sb_so) == PF_INET \|\| SOCK_DOM(sb->sb_so) == PF_INET6) {
1129	ASSERT(nodrop == FALSE);
1130
1131	if (NEED_DGRAM_FLOW_TRACKING(so)) {
1132	dgram_flow_entry = soflow_get_flow(so, NULL, NULL, NULL, m0 != NULL ? m_length(m0) : `0`, false, (m0 != NULL && m0->m_pkthdr.rcvif) ? m0->m_pkthdr.rcvif->if_index : `0`);
1133	}
1134
1135	if (sb->sb_flags & SB_RECV && !(m0 && m0->m_flags & M_SKIPCFIL)) {
1136	int error = sflt_data_in(so: sb->sb_so, NULL, data: &m0, NULL,
1137	flags: sock_data_filt_flag_record);
1138
1139	#if CONTENT_FILTER
1140	if (error == `0`) {
1141	error = cfil_sock_data_in(so: sb->sb_so, NULL, data: m0, NULL, flags: `0`, dgram_flow_entry);
1142	}
1143	#endif /* CONTENT_FILTER */
1144
1145	if (error != `0`) {
1146	SBLASTRECORDCHK(sb, "sbappendrecord 1");
1147	if (error != EJUSTRETURN) {
1148	m_freem(m0);
1149	}
1150	if (dgram_flow_entry != NULL) {
1151	soflow_free_flow(dgram_flow_entry);
1152	}
1153	return `0`;
1154	}
1155	} else if (m0) {
1156	m0->m_flags &= ~M_SKIPCFIL;
1157	}
1158
1159	if (dgram_flow_entry != NULL) {
1160	soflow_free_flow(dgram_flow_entry);
1161	}
1162	}
1163
1164	/*
1165	* Note this permits zero length records.
1166	*/
1167	sballoc(sb, m: m0);
1168	SBLASTRECORDCHK(sb, "sbappendrecord 2");
1169	if (sb->sb_lastrecord != NULL) {
1170	sb->sb_lastrecord->m_nextpkt = m0;
1171	} else {
1172	sb->sb_mb = m0;
1173	}
1174	sb->sb_lastrecord = m0;
1175	sb->sb_mbtail = m0;
1176
1177	m = m0->m_next;
1178	m0->m_next = `0`;
1179	if (m && (m0->m_flags & M_EOR)) {
1180	m0->m_flags &= ~M_EOR;
1181	m->m_flags \|= M_EOR;
1182	}
1183	sbcompress(sb, m, m0);
1184	SBLASTRECORDCHK(sb, "sbappendrecord 3");
1185	return `1`;
1186	}
1187
1188	int
1189	sbappendrecord(struct sockbuf sb, struct* mbuf *m0)
1190	{
1191	return sbappendrecord_common(sb, m0, FALSE);
1192	}
1193
1194	int
1195	sbappendrecord_nodrop(struct sockbuf sb, struct* mbuf *m0)
1196	{
1197	return sbappendrecord_common(sb, m0, TRUE);
1198	}
1199
1200	/*
1201	* Concatenate address (optional), control (optional) and data into one
1202	* single mbuf chain. If sockbuf *sb is passed in, space check will be
1203	* performed.
1204	*
1205	* Returns: mbuf chain pointer if succeeded, NULL if failed
1206	*/
1207	struct mbuf *
1208	sbconcat_mbufs(struct sockbuf sb, struct* sockaddr asa, struct* mbuf m0, struct* mbuf *control)
1209	{
1210	struct mbuf m = NULL, n = NULL;
1211	int space = `0`;
1212
1213	if (m0 && (m0->m_flags & M_PKTHDR) == `0`) {
1214	panic("sbconcat_mbufs");
1215	}
1216
1217	if (m0) {
1218	space += m0->m_pkthdr.len;
1219	}
1220	for (n = control; n; n = n->m_next) {
1221	space += n->m_len;
1222	if (n->m_next == `0`) { / keep pointer to last control buf /
1223	break;
1224	}
1225	}
1226
1227	if (asa != NULL) {
1228	_CASSERT(sizeof(asa->sa_len) == sizeof(__uint8_t));
1229	if (MLEN <= UINT8_MAX && asa->sa_len > MLEN) {
1230	return NULL;
1231	}
1232	space += asa->sa_len;
1233	}
1234
1235	if (sb != NULL && space > sbspace(sb)) {
1236	return NULL;
1237	}
1238
1239	if (n) {
1240	n->m_next = m0; / concatenate data to control /
1241	} else {
1242	control = m0;
1243	}
1244
1245	if (asa != NULL) {
1246	MGET(m, M_DONTWAIT, MT_SONAME);
1247	if (m == `0`) {
1248	if (n) {
1249	/ unchain control and data if necessary /
1250	n->m_next = NULL;
1251	}
1252	return NULL;
1253	}
1254	m->m_len = asa->sa_len;
1255	bcopy(src: (caddr_t)asa, mtod(m, caddr_t), n: asa->sa_len);
1256
1257	m->m_next = control;
1258	} else {
1259	m = control;
1260	}
1261
1262	return m;
1263	}
1264
1265	/*
1266	* Queue mbuf chain to the receive queue of a socket.
1267	* Parameter space is the total len of the mbuf chain.
1268	* If passed in, sockbuf space will be checked.
1269	*
1270	* Returns: 0 Invalid mbuf chain
1271	* 1 Success
1272	*/
1273	int
1274	sbappendchain(struct sockbuf sb, struct* mbuf m, int* space)
1275	{
1276	struct mbuf n, nlast;
1277
1278	if (m == NULL) {
1279	return `0`;
1280	}
1281
1282	if (space != `0` && space > sbspace(sb)) {
1283	return `0`;
1284	}
1285
1286	for (n = m; n->m_next != NULL; n = n->m_next) {
1287	sballoc(sb, m: n);
1288	}
1289	sballoc(sb, m: n);
1290	nlast = n;
1291
1292	if (sb->sb_lastrecord != NULL) {
1293	sb->sb_lastrecord->m_nextpkt = m;
1294	} else {
1295	sb->sb_mb = m;
1296	}
1297	sb->sb_lastrecord = m;
1298	sb->sb_mbtail = nlast;
1299
1300	SBLASTMBUFCHK(sb, __func__);
1301	SBLASTRECORDCHK(sb, "sbappendadddr 2");
1302	return `1`;
1303	}
1304
1305	/*
1306	* Returns: 0 Error: No space/out of mbufs/etc.
1307	* 1 Success
1308	*
1309	* Imputed: (*error_out) errno for error
1310	* ENOBUFS
1311	* sflt_data_in:??? [whatever a filter author chooses]
1312	*/
1313	int
1314	sbappendaddr(struct sockbuf sb, struct* sockaddr asa, struct* mbuf *m0,
1315	struct mbuf control, int* *error_out)
1316	{
1317	int result = `0`;
1318	boolean_t sb_unix = (sb->sb_flags & SB_UNIX);
1319	struct mbuf *mbuf_chain = NULL;
1320	struct soflow_hash_entry *dgram_flow_entry = NULL;
1321	struct socket *so = sb->sb_so;
1322
1323	if (error_out) {
1324	*error_out = `0`;
1325	}
1326
1327	if (m0 && (m0->m_flags & M_PKTHDR) == `0`) {
1328	panic("sbappendaddrorfree");
1329	}
1330
1331	if (sb->sb_flags & SB_DROP) {
1332	if (m0 != NULL) {
1333	m_freem(m0);
1334	}
1335	if (control != NULL && !sb_unix) {
1336	m_freem(control);
1337	}
1338	if (error_out != NULL) {
1339	*error_out = EINVAL;
1340	}
1341	return `0`;
1342	}
1343
1344	if (SOCK_DOM(sb->sb_so) == PF_INET \|\| SOCK_DOM(sb->sb_so) == PF_INET6) {
1345	/ Call socket data in filters /
1346
1347	if (NEED_DGRAM_FLOW_TRACKING(so)) {
1348	dgram_flow_entry = soflow_get_flow(so, NULL, asa, control, m0 != NULL ? m_length(m0) : `0`, false, (m0 != NULL && m0->m_pkthdr.rcvif) ? m0->m_pkthdr.rcvif->if_index : `0`);
1349	}
1350
1351	if (sb->sb_flags & SB_RECV && !(m0 && m0->m_flags & M_SKIPCFIL)) {
1352	int error;
1353	error = sflt_data_in(so: sb->sb_so, from: asa, data: &m0, control: &control, flags: `0`);
1354	SBLASTRECORDCHK(sb, __func__);
1355
1356	#if CONTENT_FILTER
1357	if (error == `0`) {
1358	error = cfil_sock_data_in(so: sb->sb_so, from: asa, data: m0, control,
1359	flags: `0`, dgram_flow_entry);
1360	}
1361	#endif /* CONTENT_FILTER */
1362
1363	if (error) {
1364	if (error != EJUSTRETURN) {
1365	if (m0) {
1366	m_freem(m0);
1367	}
1368	if (control != NULL && !sb_unix) {
1369	m_freem(control);
1370	}
1371	if (error_out) {
1372	*error_out = error;
1373	}
1374	}
1375	if (dgram_flow_entry != NULL) {
1376	soflow_free_flow(dgram_flow_entry);
1377	}
1378	return `0`;
1379	}
1380	} else if (m0) {
1381	m0->m_flags &= ~M_SKIPCFIL;
1382	}
1383
1384	if (dgram_flow_entry != NULL) {
1385	soflow_free_flow(dgram_flow_entry);
1386	}
1387	}
1388
1389	mbuf_chain = sbconcat_mbufs(sb, asa, m0, control);
1390	SBLASTRECORDCHK(sb, "sbappendadddr 1");
1391	result = sbappendchain(sb, m: mbuf_chain, space: `0`);
1392	if (result == `0`) {
1393	if (m0) {
1394	m_freem(m0);
1395	}
1396	if (control != NULL && !sb_unix) {
1397	m_freem(control);
1398	}
1399	if (error_out) {
1400	*error_out = ENOBUFS;
1401	}
1402	}
1403
1404	return result;
1405	}
1406
1407	inline boolean_t
1408	is_cmsg_valid(struct mbuf control, struct* cmsghdr *cmsg)
1409	{
1410	if (cmsg == NULL) {
1411	return FALSE;
1412	}
1413
1414	if (cmsg->cmsg_len < sizeof(struct cmsghdr)) {
1415	return FALSE;
1416	}
1417
1418	if ((uint8_t )control->m_data >= (uint8_t )cmsg + cmsg->cmsg_len) {
1419	return FALSE;
1420	}
1421
1422	if ((uint8_t *)control->m_data + control->m_len <
1423	(uint8_t *)cmsg + cmsg->cmsg_len) {
1424	return FALSE;
1425	}
1426
1427	return TRUE;
1428	}
1429
1430	static int
1431	sbappendcontrol_internal(struct sockbuf sb, struct* mbuf *m0,
1432	struct mbuf *control)
1433	{
1434	struct mbuf m, mlast, *n;
1435	int space = `0`;
1436
1437	if (control == `0`) {
1438	panic("sbappendcontrol");
1439	}
1440
1441	for (m = control;; m = m->m_next) {
1442	space += m->m_len;
1443	if (m->m_next == `0`) {
1444	break;
1445	}
1446	}
1447	n = m; / save pointer to last control buffer /
1448	for (m = m0; m; m = m->m_next) {
1449	space += m->m_len;
1450	}
1451	if (space > sbspace(sb) && !(sb->sb_flags & SB_UNIX)) {
1452	return `0`;
1453	}
1454	n->m_next = m0; / concatenate data to control /
1455	SBLASTRECORDCHK(sb, "sbappendcontrol 1");
1456
1457	for (m = control; m->m_next != NULL; m = m->m_next) {
1458	sballoc(sb, m);
1459	}
1460	sballoc(sb, m);
1461	mlast = m;
1462
1463	if (sb->sb_lastrecord != NULL) {
1464	sb->sb_lastrecord->m_nextpkt = control;
1465	} else {
1466	sb->sb_mb = control;
1467	}
1468	sb->sb_lastrecord = control;
1469	sb->sb_mbtail = mlast;
1470
1471	SBLASTMBUFCHK(sb, __func__);
1472	SBLASTRECORDCHK(sb, "sbappendcontrol 2");
1473	return `1`;
1474	}
1475
1476	int
1477	sbappendcontrol(struct sockbuf sb, struct* mbuf m0, struct* mbuf *control,
1478	int *error_out)
1479	{
1480	struct soflow_hash_entry *dgram_flow_entry = NULL;
1481	struct socket *so = sb->sb_so;
1482	int result = `0`;
1483	boolean_t sb_unix = (sb->sb_flags & SB_UNIX);
1484
1485	if (error_out) {
1486	*error_out = `0`;
1487	}
1488
1489	if (sb->sb_flags & SB_DROP) {
1490	if (m0 != NULL) {
1491	m_freem(m0);
1492	}
1493	if (control != NULL && !sb_unix) {
1494	m_freem(control);
1495	}
1496	if (error_out != NULL) {
1497	*error_out = EINVAL;
1498	}
1499	return `0`;
1500	}
1501
1502	if (SOCK_DOM(sb->sb_so) == PF_INET \|\| SOCK_DOM(sb->sb_so) == PF_INET6) {
1503	if (NEED_DGRAM_FLOW_TRACKING(so)) {
1504	dgram_flow_entry = soflow_get_flow(so, NULL, NULL, control, m0 != NULL ? m_length(m0) : `0`, false, (m0 != NULL && m0->m_pkthdr.rcvif) ? m0->m_pkthdr.rcvif->if_index : `0`);
1505	}
1506
1507	if (sb->sb_flags & SB_RECV && !(m0 && m0->m_flags & M_SKIPCFIL)) {
1508	int error;
1509
1510	error = sflt_data_in(so: sb->sb_so, NULL, data: &m0, control: &control, flags: `0`);
1511	SBLASTRECORDCHK(sb, __func__);
1512
1513	#if CONTENT_FILTER
1514	if (error == `0`) {
1515	error = cfil_sock_data_in(so: sb->sb_so, NULL, data: m0, control,
1516	flags: `0`, dgram_flow_entry);
1517	}
1518	#endif /* CONTENT_FILTER */
1519
1520	if (error) {
1521	if (error != EJUSTRETURN) {
1522	if (m0) {
1523	m_freem(m0);
1524	}
1525	if (control != NULL && !sb_unix) {
1526	m_freem(control);
1527	}
1528	if (error_out) {
1529	*error_out = error;
1530	}
1531	}
1532	if (dgram_flow_entry != NULL) {
1533	soflow_free_flow(dgram_flow_entry);
1534	}
1535	return `0`;
1536	}
1537	} else if (m0) {
1538	m0->m_flags &= ~M_SKIPCFIL;
1539	}
1540
1541	if (dgram_flow_entry != NULL) {
1542	soflow_free_flow(dgram_flow_entry);
1543	}
1544	}
1545
1546	result = sbappendcontrol_internal(sb, m0, control);
1547	if (result == `0`) {
1548	if (m0) {
1549	m_freem(m0);
1550	}
1551	if (control != NULL && !sb_unix) {
1552	m_freem(control);
1553	}
1554	if (error_out) {
1555	*error_out = ENOBUFS;
1556	}
1557	}
1558
1559	return result;
1560	}
1561
1562	/*
1563	* TCP streams have Multipath TCP support or are regular TCP sockets.
1564	*/
1565	int
1566	sbappendstream_rcvdemux(struct socket so, struct* mbuf *m)
1567	{
1568	int ret = `0`;
1569
1570	if ((m != NULL) &&
1571	m_pktlen(m) <= `0` &&
1572	!((so->so_flags & SOF_MP_SUBFLOW) &&
1573	(m->m_flags & M_PKTHDR) &&
1574	(m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN))) {
1575	m_freem(m);
1576	return ret;
1577	}
1578
1579	#if MPTCP
1580	if (so->so_flags & SOF_MP_SUBFLOW) {
1581	return sbappendmptcpstream_rcv(sb: &so->so_rcv, m);
1582	} else
1583	#endif /* MPTCP */
1584	{
1585	return sbappendstream(sb: &so->so_rcv, m);
1586	}
1587	}
1588
1589	#if MPTCP
1590	int
1591	sbappendmptcpstream_rcv(struct sockbuf sb, struct* mbuf *m)
1592	{
1593	struct socket *so = sb->sb_so;
1594
1595	VERIFY(m == NULL \|\| (m->m_flags & M_PKTHDR));
1596	/ SB_NOCOMPRESS must be set prevent loss of M_PKTHDR data /
1597	VERIFY((sb->sb_flags & (SB_RECV \| SB_NOCOMPRESS)) ==
1598	(SB_RECV \| SB_NOCOMPRESS));
1599
1600	if (m == NULL \|\| m_pktlen(m) == `0` \|\| (sb->sb_flags & SB_DROP) \|\|
1601	(so->so_state & SS_CANTRCVMORE)) {
1602	if (m && (m->m_flags & M_PKTHDR) &&
1603	m_pktlen(m) == `0` &&
1604	(m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)) {
1605	mptcp_input(tptomptp(sototcpcb(so))->mpt_mpte, m);
1606	return `1`;
1607	} else if (m != NULL) {
1608	m_freem(m);
1609	}
1610	return `0`;
1611	}
1612	/ the socket is not closed, so SOF_MP_SUBFLOW must be set /
1613	VERIFY(so->so_flags & SOF_MP_SUBFLOW);
1614
1615	if (m->m_nextpkt != NULL \|\| (sb->sb_mb != sb->sb_lastrecord)) {
1616	panic("%s: nexpkt %p \|\| mb %p != lastrecord %p", __func__,
1617	m->m_nextpkt, sb->sb_mb, sb->sb_lastrecord);
1618	/ NOTREACHED /
1619	}
1620
1621	SBLASTMBUFCHK(sb, __func__);
1622
1623	/ No filter support (SB_RECV) on mptcp subflow sockets /
1624
1625	sbcompress(sb, m, sb->sb_mbtail);
1626	sb->sb_lastrecord = sb->sb_mb;
1627	SBLASTRECORDCHK(sb, __func__);
1628	return `1`;
1629	}
1630	#endif /* MPTCP */
1631
1632	/*
1633	* Compress mbuf chain m into the socket
1634	* buffer sb following mbuf n. If n
1635	* is null, the buffer is presumed empty.
1636	*/
1637	static inline void
1638	sbcompress(struct sockbuf sb, struct* mbuf m, struct* mbuf *n)
1639	{
1640	int eor = `0`, compress = (!(sb->sb_flags & SB_NOCOMPRESS));
1641	struct mbuf *o;
1642
1643	if (m == NULL) {
1644	/ There is nothing to compress; just update the tail /
1645	for (; n->m_next != NULL; n = n->m_next) {
1646	;
1647	}
1648	sb->sb_mbtail = n;
1649	goto done;
1650	}
1651
1652	while (m != NULL) {
1653	eor \|= m->m_flags & M_EOR;
1654	if (compress && m->m_len == `0` && (eor == `0` \|\|
1655	(((o = m->m_next) \|\| (o = n)) && o->m_type == m->m_type))) {
1656	if (sb->sb_lastrecord == m) {
1657	sb->sb_lastrecord = m->m_next;
1658	}
1659	m = m_free(m);
1660	continue;
1661	}
1662	if (compress && n != NULL && (n->m_flags & M_EOR) == `0` &&
1663	#ifndef __APPLE__
1664	M_WRITABLE(n) &&
1665	#endif
1666	m->m_len <= MCLBYTES / `4` && / XXX: Don't copy too much /
1667	m->m_len <= M_TRAILINGSPACE(n) &&
1668	n->m_type == m->m_type) {
1669	bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len,
1670	n: (unsigned)m->m_len);
1671	n->m_len += m->m_len;
1672	sb->sb_cc += m->m_len;
1673	if (!m_has_mtype(m, mtype_flags: MTF_DATA \| MTF_HEADER \| MTF_OOBDATA)) {
1674	/ XXX: Probably don't need /
1675	sb->sb_ctl += m->m_len;
1676	}
1677
1678	/ update send byte count /
1679	if (sb->sb_flags & SB_SNDBYTE_CNT) {
1680	inp_incr_sndbytes_total(sb->sb_so,
1681	m->m_len);
1682	inp_incr_sndbytes_unsent(sb->sb_so,
1683	m->m_len);
1684	}
1685	m = m_free(m);
1686	continue;
1687	}
1688	if (n != NULL) {
1689	n->m_next = m;
1690	} else {
1691	sb->sb_mb = m;
1692	}
1693	sb->sb_mbtail = m;
1694	sballoc(sb, m);
1695	n = m;
1696	m->m_flags &= ~M_EOR;
1697	m = m->m_next;
1698	n->m_next = NULL;
1699	}
1700	if (eor != `0`) {
1701	if (n != NULL) {
1702	n->m_flags \|= M_EOR;
1703	} else {
1704	printf("semi-panic: sbcompress\n");
1705	}
1706	}
1707	done:
1708	SBLASTMBUFCHK(sb, __func__);
1709	}
1710
1711	void
1712	sb_empty_assert(struct sockbuf sb, const* char *where)
1713	{
1714	if (!(sb->sb_cc == `0` && sb->sb_mb == NULL && sb->sb_mbcnt == `0` &&
1715	sb->sb_mbtail == NULL && sb->sb_lastrecord == NULL)) {
1716	panic("%s: sb %p so %p cc %d mbcnt %d mb %p mbtail %p "
1717	"lastrecord %p\n", where, sb, sb->sb_so, sb->sb_cc,
1718	sb->sb_mbcnt, sb->sb_mb, sb->sb_mbtail,
1719	sb->sb_lastrecord);
1720	/ NOTREACHED /
1721	}
1722	}
1723
1724	/*
1725	* Free all mbufs in a sockbuf.
1726	* Check that all resources are reclaimed.
1727	*/
1728	void
1729	sbflush(struct sockbuf *sb)
1730	{
1731	void *lr_saved = __builtin_return_address(`0`);
1732	struct socket *so = sb->sb_so;
1733
1734	/ so_usecount may be 0 if we get here from sofreelastref() /
1735	if (so == NULL) {
1736	panic("%s: null so, sb=%p sb_flags=0x%x lr=%p",
1737	__func__, sb, sb->sb_flags, lr_saved);
1738	/ NOTREACHED /
1739	} else if (so->so_usecount < `0`) {
1740	panic("%s: sb=%p sb_flags=0x%x sb_so=%p usecount=%d lr=%p "
1741	"lrh= %s\n", __func__, sb, sb->sb_flags, so,
1742	so->so_usecount, lr_saved, solockhistory_nr(so));
1743	/ NOTREACHED /
1744	}
1745
1746	/*
1747	* Obtain lock on the socket buffer (SB_LOCK). This is required
1748	* to prevent the socket buffer from being unexpectedly altered
1749	* while it is used by another thread in socket send/receive.
1750	*
1751	* sblock() must not fail here, hence the assertion.
1752	*/
1753	(void) sblock(sb, SBL_WAIT \| SBL_NOINTR \| SBL_IGNDEFUNCT);
1754	VERIFY(sb->sb_flags & SB_LOCK);
1755
1756	while (sb->sb_mbcnt > `0`) {
1757	/*
1758	* Don't call sbdrop(sb, 0) if the leading mbuf is non-empty:
1759	* we would loop forever. Panic instead.
1760	*/
1761	if (!sb->sb_cc && (sb->sb_mb == NULL \|\| sb->sb_mb->m_len)) {
1762	break;
1763	}
1764	sbdrop(sb, len: (int)sb->sb_cc);
1765	}
1766
1767	if (sb->sb_flags & SB_SENDHEAD) {
1768	sb->sb_sendhead = NULL;
1769	}
1770
1771	sb_empty_assert(sb, where: __func__);
1772	sbunlock(sb, TRUE); / keep socket locked /
1773	}
1774
1775	/*
1776	* Drop data from (the front of) a sockbuf.
1777	* use m_freem_list to free the mbuf structures
1778	* under a single lock... this is done by pruning
1779	* the top of the tree from the body by keeping track
1780	* of where we get to in the tree and then zeroing the
1781	* two pertinent pointers m_nextpkt and m_next
1782	* the socket buffer is then updated to point at the new
1783	* top of the tree and the pruned area is released via
1784	* m_freem_list.
1785	*/
1786	void
1787	sbdrop(struct sockbuf sb, int* len)
1788	{
1789	struct mbuf m, free_list, *ml;
1790	struct mbuf next, last;
1791
1792	next = (m = sb->sb_mb) ? m->m_nextpkt : `0`;
1793	#if MPTCP
1794	if (m != NULL && len > `0` && !(sb->sb_flags & SB_RECV) &&
1795	((sb->sb_so->so_flags & SOF_MP_SUBFLOW) \|\|
1796	(SOCK_CHECK_DOM(sb->sb_so, PF_MULTIPATH) &&
1797	SOCK_CHECK_PROTO(sb->sb_so, IPPROTO_TCP))) &&
1798	!(sb->sb_so->so_flags1 & SOF1_POST_FALLBACK_SYNC)) {
1799	mptcp_preproc_sbdrop(sb->sb_so, m, (unsigned int)len);
1800	}
1801	if (m != NULL && len > `0` && !(sb->sb_flags & SB_RECV) &&
1802	(sb->sb_so->so_flags & SOF_MP_SUBFLOW) &&
1803	(sb->sb_so->so_flags1 & SOF1_POST_FALLBACK_SYNC)) {
1804	mptcp_fallback_sbdrop(so: sb->sb_so, m, len);
1805	}
1806	#endif /* MPTCP */
1807	KERNEL_DEBUG((DBG_FNC_SBDROP \| DBG_FUNC_START), sb, len, `0`, `0`, `0`);
1808
1809	free_list = last = m;
1810	ml = (struct mbuf *)`0`;
1811
1812	if (sb->sb_flags & SB_SENDHEAD) {
1813	sb->sb_sendoff -= MIN(len, sb->sb_sendoff);
1814	}
1815
1816	while (len > `0`) {
1817	if (m == NULL) {
1818	if (next == NULL) {
1819	/*
1820	* temporarily replacing this panic with printf
1821	* because it occurs occasionally when closing
1822	* a socket when there is no harm in ignoring
1823	* it. This problem will be investigated
1824	* further.
1825	*/
1826	/ panic("sbdrop"); /
1827	printf("sbdrop - count not zero\n");
1828	len = `0`;
1829	/*
1830	* zero the counts. if we have no mbufs,
1831	* we have no data (PR-2986815)
1832	*/
1833	sb->sb_cc = `0`;
1834	sb->sb_mbcnt = `0`;
1835	break;
1836	}
1837	m = last = next;
1838	next = m->m_nextpkt;
1839	continue;
1840	}
1841	if (m->m_len > len) {
1842	m->m_len -= len;
1843	m->m_data += len;
1844	sb->sb_cc -= len;
1845	/ update the send byte count /
1846	if (sb->sb_flags & SB_SNDBYTE_CNT) {
1847	inp_decr_sndbytes_total(sb->sb_so, len);
1848	}
1849	if (sb->sb_flags & SB_SENDHEAD) {
1850	if (sb->sb_sendhead == m) {
1851	sb->sb_sendhead = NULL;
1852	}
1853	}
1854	if (!m_has_mtype(m, mtype_flags: MTF_DATA \| MTF_HEADER \| MTF_OOBDATA)) {
1855	sb->sb_ctl -= len;
1856	}
1857	break;
1858	}
1859	len -= m->m_len;
1860	sbfree(sb, m);
1861
1862	ml = m;
1863	m = m->m_next;
1864	}
1865	while (m && m->m_len == `0`) {
1866	sbfree(sb, m);
1867
1868	ml = m;
1869	m = m->m_next;
1870	}
1871	if (ml) {
1872	ml->m_next = (struct mbuf *)`0`;
1873	last->m_nextpkt = (struct mbuf *)`0`;
1874	m_freem_list(free_list);
1875	}
1876	if (m) {
1877	sb->sb_mb = m;
1878	m->m_nextpkt = next;
1879	} else {
1880	sb->sb_mb = next;
1881	}
1882
1883	/*
1884	* First part is an inline SB_EMPTY_FIXUP(). Second part
1885	* makes sure sb_lastrecord is up-to-date if we dropped
1886	* part of the last record.
1887	*/
1888	m = sb->sb_mb;
1889	if (m == NULL) {
1890	sb->sb_mbtail = NULL;
1891	sb->sb_lastrecord = NULL;
1892	} else if (m->m_nextpkt == NULL) {
1893	sb->sb_lastrecord = m;
1894	}
1895
1896	#if CONTENT_FILTER
1897	cfil_sock_buf_update(sb);
1898	#endif /* CONTENT_FILTER */
1899
1900	KERNEL_DEBUG((DBG_FNC_SBDROP \| DBG_FUNC_END), sb, `0`, `0`, `0`, `0`);
1901	}
1902
1903	/*
1904	* Drop a record off the front of a sockbuf
1905	* and move the next record to the front.
1906	*/
1907	void
1908	sbdroprecord(struct sockbuf *sb)
1909	{
1910	struct mbuf m, mn;
1911
1912	m = sb->sb_mb;
1913	if (m) {
1914	sb->sb_mb = m->m_nextpkt;
1915	do {
1916	sbfree(sb, m);
1917	MFREE(m, mn);
1918	m = mn;
1919	} while (m);
1920	}
1921	SB_EMPTY_FIXUP(sb);
1922	}
1923
1924	/*
1925	* Create a "control" mbuf containing the specified data
1926	* with the specified type for presentation on a socket buffer.
1927	*/
1928	struct mbuf *
1929	sbcreatecontrol(caddr_t p, int size, int type, int level)
1930	{
1931	struct cmsghdr *cp;
1932	struct mbuf *m;
1933
1934	if (CMSG_SPACE((u_int)size) > MLEN) {
1935	return (struct mbuf *)NULL;
1936	}
1937	if ((m = m_get(M_DONTWAIT, MT_CONTROL)) == NULL) {
1938	return (struct mbuf *)NULL;
1939	}
1940	cp = mtod(m, struct cmsghdr *);
1941	VERIFY(IS_P2ALIGNED(cp, sizeof(u_int32_t)));
1942	/ XXX check size? /
1943	(void) memcpy(CMSG_DATA(cp), src: p, n: size);
1944	m->m_len = (int32_t)CMSG_SPACE(size);
1945	cp->cmsg_len = CMSG_LEN(size);
1946	cp->cmsg_level = level;
1947	cp->cmsg_type = type;
1948	return m;
1949	}
1950
1951	struct mbuf **
1952	sbcreatecontrol_mbuf(caddr_t p, int size, int type, int level, struct mbuf **mp)
1953	{
1954	struct mbuf *m;
1955	struct cmsghdr *cp;
1956
1957	if (*mp == NULL) {
1958	*mp = sbcreatecontrol(p, size, type, level);
1959	return mp;
1960	}
1961
1962	if (CMSG_SPACE((u_int)size) + (*mp)->m_len > MLEN) {
1963	mp = &(*mp)->m_next;
1964	*mp = sbcreatecontrol(p, size, type, level);
1965	return mp;
1966	}
1967
1968	m = *mp;
1969
1970	cp = (struct cmsghdr )(void* )(mtod(m, char* *) + m->m_len);
1971	/ CMSG_SPACE ensures 32-bit alignment /
1972	VERIFY(IS_P2ALIGNED(cp, sizeof(u_int32_t)));
1973	m->m_len += (int32_t)CMSG_SPACE(size);
1974
1975	/ XXX check size? /
1976	(void) memcpy(CMSG_DATA(cp), src: p, n: size);
1977	cp->cmsg_len = CMSG_LEN(size);
1978	cp->cmsg_level = level;
1979	cp->cmsg_type = type;
1980
1981	return mp;
1982	}
1983
1984
1985	/*
1986	* Some routines that return EOPNOTSUPP for entry points that are not
1987	* supported by a protocol. Fill in as needed.
1988	*/
1989	int
1990	pru_abort_notsupp(struct socket *so)
1991	{
1992	#pragma unused(so)
1993	return EOPNOTSUPP;
1994	}
1995
1996	int
1997	pru_accept_notsupp(struct socket so, struct* sockaddr **nam)
1998	{
1999	#pragma unused(so, nam)
2000	return EOPNOTSUPP;
2001	}
2002
2003	int
2004	pru_attach_notsupp(struct socket so, int* proto, struct proc *p)
2005	{
2006	#pragma unused(so, proto, p)
2007	return EOPNOTSUPP;
2008	}
2009
2010	int
2011	pru_bind_notsupp(struct socket so, struct* sockaddr nam, struct* proc *p)
2012	{
2013	#pragma unused(so, nam, p)
2014	return EOPNOTSUPP;
2015	}
2016
2017	int
2018	pru_connect_notsupp(struct socket so, struct* sockaddr nam, struct* proc *p)
2019	{
2020	#pragma unused(so, nam, p)
2021	return EOPNOTSUPP;
2022	}
2023
2024	int
2025	pru_connect2_notsupp(struct socket so1, struct* socket *so2)
2026	{
2027	#pragma unused(so1, so2)
2028	return EOPNOTSUPP;
2029	}
2030
2031	int
2032	pru_connectx_notsupp(struct socket so, struct* sockaddr *src,
2033	struct sockaddr dst, struct* proc *p, uint32_t ifscope,
2034	sae_associd_t aid, sae_connid_t pcid, uint32_t flags, void* *arg,
2035	uint32_t arglen, struct uio uio, user_ssize_t bytes_written)
2036	{
2037	#pragma unused(so, src, dst, p, ifscope, aid, pcid, flags, arg, arglen, uio, bytes_written)
2038	return EOPNOTSUPP;
2039	}
2040
2041	int
2042	pru_control_notsupp(struct socket *so, u_long cmd, caddr_t data,
2043	struct ifnet ifp, struct* proc *p)
2044	{
2045	#pragma unused(so, cmd, data, ifp, p)
2046	return EOPNOTSUPP;
2047	}
2048
2049	int
2050	pru_detach_notsupp(struct socket *so)
2051	{
2052	#pragma unused(so)
2053	return EOPNOTSUPP;
2054	}
2055
2056	int
2057	pru_disconnect_notsupp(struct socket *so)
2058	{
2059	#pragma unused(so)
2060	return EOPNOTSUPP;
2061	}
2062
2063	int
2064	pru_disconnectx_notsupp(struct socket *so, sae_associd_t aid, sae_connid_t cid)
2065	{
2066	#pragma unused(so, aid, cid)
2067	return EOPNOTSUPP;
2068	}
2069
2070	int
2071	pru_listen_notsupp(struct socket so, struct* proc *p)
2072	{
2073	#pragma unused(so, p)
2074	return EOPNOTSUPP;
2075	}
2076
2077	int
2078	pru_peeraddr_notsupp(struct socket so, struct* sockaddr **nam)
2079	{
2080	#pragma unused(so, nam)
2081	return EOPNOTSUPP;
2082	}
2083
2084	int
2085	pru_rcvd_notsupp(struct socket so, int* flags)
2086	{
2087	#pragma unused(so, flags)
2088	return EOPNOTSUPP;
2089	}
2090
2091	int
2092	pru_rcvoob_notsupp(struct socket so, struct* mbuf m, int* flags)
2093	{
2094	#pragma unused(so, m, flags)
2095	return EOPNOTSUPP;
2096	}
2097
2098	int
2099	pru_send_notsupp(struct socket so, int* flags, struct mbuf *m,
2100	struct sockaddr addr, struct* mbuf control, struct* proc *p)
2101	{
2102	#pragma unused(so, flags, m, addr, control, p)
2103	return EOPNOTSUPP;
2104	}
2105
2106	int
2107	pru_send_list_notsupp(struct socket so, struct* mbuf m, u_int pktcnt,
2108	int flags)
2109	{
2110	#pragma unused(so, m, pktcnt, flags)
2111	return EOPNOTSUPP;
2112	}
2113
2114	/*
2115	* This isn't really a ``null'' operation, but it's the default one
2116	* and doesn't do anything destructive.
2117	*/
2118	int
2119	pru_sense_null(struct socket so, void* ub, int* isstat64)
2120	{
2121	if (isstat64 != `0`) {
2122	struct stat64 *sb64;
2123
2124	sb64 = (struct stat64 *)ub;
2125	sb64->st_blksize = so->so_snd.sb_hiwat;
2126	} else {
2127	struct stat *sb;
2128
2129	sb = (struct stat *)ub;
2130	sb->st_blksize = so->so_snd.sb_hiwat;
2131	}
2132
2133	return `0`;
2134	}
2135
2136	int
2137	pru_sosend_notsupp(struct socket so, struct* sockaddr addr, struct* uio *uio,
2138	struct mbuf top, struct* mbuf control, int* flags)
2139	{
2140	#pragma unused(so, addr, uio, top, control, flags)
2141	return EOPNOTSUPP;
2142	}
2143
2144	int
2145	pru_sosend_list_notsupp(struct socket so, struct* mbuf m, size_t total_len, u_int pktcnt, int flags)
2146	{
2147	#pragma unused(so, m, total_len, pktcnt, flags)
2148	return EOPNOTSUPP;
2149	}
2150
2151	int
2152	pru_soreceive_notsupp(struct socket so, struct* sockaddr **paddr,
2153	struct uio uio, struct* mbuf mp0, struct mbuf controlp, int *flagsp)
2154	{
2155	#pragma unused(so, paddr, uio, mp0, controlp, flagsp)
2156	return EOPNOTSUPP;
2157	}
2158
2159	int
2160	pru_shutdown_notsupp(struct socket *so)
2161	{
2162	#pragma unused(so)
2163	return EOPNOTSUPP;
2164	}
2165
2166	int
2167	pru_sockaddr_notsupp(struct socket so, struct* sockaddr **nam)
2168	{
2169	#pragma unused(so, nam)
2170	return EOPNOTSUPP;
2171	}
2172
2173	int
2174	pru_sopoll_notsupp(struct socket so, int* events, kauth_cred_t cred, void *wql)
2175	{
2176	#pragma unused(so, events, cred, wql)
2177	return EOPNOTSUPP;
2178	}
2179
2180	int
2181	pru_socheckopt_null(struct socket so, struct* sockopt *sopt)
2182	{
2183	#pragma unused(so, sopt)
2184	/*
2185	* Allow all options for set/get by default.
2186	*/
2187	return `0`;
2188	}
2189
2190	static int
2191	pru_preconnect_null(struct socket *so)
2192	{
2193	#pragma unused(so)
2194	return `0`;
2195	}
2196
2197	static int
2198	pru_defunct_null(struct socket *so)
2199	{
2200	#pragma unused(so)
2201	return `0`;
2202	}
2203
2204
2205	void
2206	pru_sanitize(struct pr_usrreqs *pru)
2207	{
2208	#define DEFAULT(foo, bar) if ((foo) == NULL) (foo) = (bar)
2209	DEFAULT(pru->pru_abort, pru_abort_notsupp);
2210	DEFAULT(pru->pru_accept, pru_accept_notsupp);
2211	DEFAULT(pru->pru_attach, pru_attach_notsupp);
2212	DEFAULT(pru->pru_bind, pru_bind_notsupp);
2213	DEFAULT(pru->pru_connect, pru_connect_notsupp);
2214	DEFAULT(pru->pru_connect2, pru_connect2_notsupp);
2215	DEFAULT(pru->pru_connectx, pru_connectx_notsupp);
2216	DEFAULT(pru->pru_control, pru_control_notsupp);
2217	DEFAULT(pru->pru_detach, pru_detach_notsupp);
2218	DEFAULT(pru->pru_disconnect, pru_disconnect_notsupp);
2219	DEFAULT(pru->pru_disconnectx, pru_disconnectx_notsupp);
2220	DEFAULT(pru->pru_listen, pru_listen_notsupp);
2221	DEFAULT(pru->pru_peeraddr, pru_peeraddr_notsupp);
2222	DEFAULT(pru->pru_rcvd, pru_rcvd_notsupp);
2223	DEFAULT(pru->pru_rcvoob, pru_rcvoob_notsupp);
2224	DEFAULT(pru->pru_send, pru_send_notsupp);
2225	DEFAULT(pru->pru_send_list, pru_send_list_notsupp);
2226	DEFAULT(pru->pru_sense, pru_sense_null);
2227	DEFAULT(pru->pru_shutdown, pru_shutdown_notsupp);
2228	DEFAULT(pru->pru_sockaddr, pru_sockaddr_notsupp);
2229	DEFAULT(pru->pru_sopoll, pru_sopoll_notsupp);
2230	DEFAULT(pru->pru_soreceive, pru_soreceive_notsupp);
2231	DEFAULT(pru->pru_sosend, pru_sosend_notsupp);
2232	DEFAULT(pru->pru_sosend_list, pru_sosend_list_notsupp);
2233	DEFAULT(pru->pru_socheckopt, pru_socheckopt_null);
2234	DEFAULT(pru->pru_preconnect, pru_preconnect_null);
2235	DEFAULT(pru->pru_defunct, pru_defunct_null);
2236	#undef DEFAULT
2237	}
2238
2239	/*
2240	* The following are macros on BSD and functions on Darwin
2241	*/
2242
2243	/*
2244	* Do we need to notify the other side when I/O is possible?
2245	*/
2246
2247	int
2248	sb_notify(struct sockbuf *sb)
2249	{
2250	return sb->sb_waiters > `0` \|\|
2251	(sb->sb_flags & (SB_SEL \| SB_ASYNC \| SB_UPCALL \| SB_KNOTE));
2252	}
2253
2254	/*
2255	* How much space is there in a socket buffer (so->so_snd or so->so_rcv)?
2256	* This is problematical if the fields are unsigned, as the space might
2257	* still be negative (cc > hiwat or mbcnt > mbmax). Should detect
2258	* overflow and return 0.
2259	*/
2260	int
2261	sbspace(struct sockbuf *sb)
2262	{
2263	int pending = `0`;
2264	int space;
2265
2266	if (sb->sb_flags & SB_KCTL) {
2267	space = (int)(sb->sb_hiwat - sb->sb_cc);
2268	} else {
2269	space = imin(a: (int)(sb->sb_hiwat - sb->sb_cc),
2270	b: (int)(sb->sb_mbmax - sb->sb_mbcnt));
2271	}
2272	if (sb->sb_preconn_hiwat != `0`) {
2273	space = imin(a: (int)(sb->sb_preconn_hiwat - sb->sb_cc), b: space);
2274	}
2275
2276	if (space < `0`) {
2277	space = `0`;
2278	}
2279
2280	/ Compensate for data being processed by content filters /
2281	#if CONTENT_FILTER
2282	pending = cfil_sock_data_space(sb);
2283	#endif /* CONTENT_FILTER */
2284	if (pending > space) {
2285	space = `0`;
2286	} else {
2287	space -= pending;
2288	}
2289
2290	return space;
2291	}
2292
2293	/ do we have to send all at once on a socket? /
2294	int
2295	sosendallatonce(struct socket *so)
2296	{
2297	return so->so_proto->pr_flags & PR_ATOMIC;
2298	}
2299
2300	/ can we read something from so? /
2301	int
2302	soreadable(struct socket *so)
2303	{
2304	return so->so_rcv.sb_cc >= so->so_rcv.sb_lowat \|\|
2305	((so->so_state & SS_CANTRCVMORE)
2306	#if CONTENT_FILTER
2307	&& cfil_sock_data_pending(sb: &so->so_rcv) == `0`
2308	#endif /* CONTENT_FILTER */
2309	) \|\|
2310	so->so_comp.tqh_first \|\| so->so_error;
2311	}
2312
2313	/ can we write something to so? /
2314
2315	int
2316	sowriteable(struct socket *so)
2317	{
2318	if ((so->so_state & SS_CANTSENDMORE) \|\|
2319	so->so_error > `0`) {
2320	return `1`;
2321	}
2322	if (so_wait_for_if_feedback(so) \|\| !socanwrite(so)) {
2323	return `0`;
2324	}
2325	if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2326	return `1`;
2327	}
2328
2329	int64_t data = sbspace(sb: &so->so_snd);
2330	int64_t lowat = so->so_snd.sb_lowat;
2331	/*
2332	* Deal with connected UNIX domain sockets which
2333	* rely on the fact that the sender's socket buffer is
2334	* actually the receiver's socket buffer.
2335	*/
2336	if (SOCK_DOM(so) == PF_LOCAL) {
2337	struct unpcb *unp = sotounpcb(so);
2338	if (unp != NULL && unp->unp_conn != NULL &&
2339	unp->unp_conn->unp_socket != NULL) {
2340	struct socket *so2 = unp->unp_conn->unp_socket;
2341	/*
2342	* At this point we know that `so' is locked
2343	* and that `unp_conn` isn't going to change.
2344	* However, we don't lock `so2` because doing so
2345	* may require unlocking `so'
2346	* (see unp_get_locks_in_order()).
2347	*
2348	* Two cases can happen:
2349	*
2350	* 1) we return 1 and tell the application that
2351	* it can write. Meanwhile, another thread
2352	* fills up the socket buffer. This will either
2353	* lead to a blocking send or EWOULDBLOCK
2354	* which the application should deal with.
2355	* 2) we return 0 and tell the application that
2356	* the socket is not writable. Meanwhile,
2357	* another thread depletes the receive socket
2358	* buffer. In this case the application will
2359	* be woken up by sb_notify().
2360	*
2361	* MIN() is required because otherwise sosendcheck()
2362	* may return EWOULDBLOCK since it only considers
2363	* so->so_snd.
2364	*/
2365	data = MIN(data, sbspace(&so2->so_rcv));
2366	}
2367	}
2368
2369	if (data >= lowat) {
2370	if (so->so_flags & SOF_NOTSENT_LOWAT) {
2371	if ((SOCK_DOM(so) == PF_INET6 \|\|
2372	SOCK_DOM(so) == PF_INET) &&
2373	so->so_type == SOCK_STREAM) {
2374	return tcp_notsent_lowat_check(so);
2375	}
2376	#if MPTCP
2377	else if ((SOCK_DOM(so) == PF_MULTIPATH) &&
2378	(SOCK_PROTO(so) == IPPROTO_TCP)) {
2379	return mptcp_notsent_lowat_check(so);
2380	}
2381	#endif
2382	else {
2383	return `1`;
2384	}
2385	} else {
2386	return `1`;
2387	}
2388	}
2389	return `0`;
2390	}
2391
2392	/ adjust counters in sb reflecting allocation of m /
2393
2394	void
2395	sballoc(struct sockbuf sb, struct* mbuf *m)
2396	{
2397	sb->sb_cc += m->m_len;
2398	if (!m_has_mtype(m, mtype_flags: MTF_DATA \| MTF_HEADER \| MTF_OOBDATA)) {
2399	sb->sb_ctl += m->m_len;
2400	}
2401	sb->sb_mbcnt += _MSIZE;
2402
2403	if (m->m_flags & M_EXT) {
2404	sb->sb_mbcnt += m->m_ext.ext_size;
2405	}
2406
2407	/*
2408	* If data is being added to the send socket buffer,
2409	* update the send byte count
2410	*/
2411	if (sb->sb_flags & SB_SNDBYTE_CNT) {
2412	inp_incr_sndbytes_total(sb->sb_so, m->m_len);
2413	inp_incr_sndbytes_unsent(sb->sb_so, m->m_len);
2414	}
2415	}
2416
2417	/ adjust counters in sb reflecting freeing of m /
2418	void
2419	sbfree(struct sockbuf sb, struct* mbuf *m)
2420	{
2421	sb->sb_cc -= m->m_len;
2422	if (!m_has_mtype(m, mtype_flags: MTF_DATA \| MTF_HEADER \| MTF_OOBDATA)) {
2423	sb->sb_ctl -= m->m_len;
2424	}
2425	sb->sb_mbcnt -= _MSIZE;
2426	if (m->m_flags & M_EXT) {
2427	sb->sb_mbcnt -= m->m_ext.ext_size;
2428	}
2429
2430	/*
2431	* If data is being removed from the send socket buffer,
2432	* update the send byte count
2433	*/
2434	if (sb->sb_flags & SB_SNDBYTE_CNT) {
2435	inp_decr_sndbytes_total(sb->sb_so, m->m_len);
2436	}
2437
2438	if (sb->sb_flags & SB_SENDHEAD) {
2439	if (m == sb->sb_sendhead) {
2440	sb->sb_sendhead = NULL;
2441	}
2442	}
2443	}
2444
2445	/*
2446	* Set lock on sockbuf sb; sleep if lock is already held.
2447	* Unless SB_NOINTR is set on sockbuf, sleep is interruptible.
2448	* Returns error without lock if sleep is interrupted.
2449	*/
2450	int
2451	sblock(struct sockbuf *sb, uint32_t flags)
2452	{
2453	boolean_t nointr = ((sb->sb_flags & SB_NOINTR) \|\| (flags & SBL_NOINTR));
2454	void *lr_saved = __builtin_return_address(`0`);
2455	struct socket *so = sb->sb_so;
2456	void * wchan;
2457	int error = `0`;
2458	thread_t tp = current_thread();
2459
2460	VERIFY((flags & SBL_VALID) == flags);
2461
2462	/ so_usecount may be 0 if we get here from sofreelastref() /
2463	if (so == NULL) {
2464	panic("%s: null so, sb=%p sb_flags=0x%x lr=%p",
2465	__func__, sb, sb->sb_flags, lr_saved);
2466	/ NOTREACHED /
2467	} else if (so->so_usecount < `0`) {
2468	panic("%s: sb=%p sb_flags=0x%x sb_so=%p usecount=%d lr=%p "
2469	"lrh= %s\n", __func__, sb, sb->sb_flags, so,
2470	so->so_usecount, lr_saved, solockhistory_nr(so));
2471	/ NOTREACHED /
2472	}
2473
2474	/*
2475	* The content filter thread must hold the sockbuf lock
2476	*/
2477	if ((so->so_flags & SOF_CONTENT_FILTER) && sb->sb_cfil_thread == tp) {
2478	/*
2479	* Don't panic if we are defunct because SB_LOCK has
2480	* been cleared by sodefunct()
2481	*/
2482	if (!(so->so_flags & SOF_DEFUNCT) && !(sb->sb_flags & SB_LOCK)) {
2483	panic("%s: SB_LOCK not held for %p",
2484	__func__, sb);
2485	}
2486
2487	/ Keep the sockbuf locked /
2488	return `0`;
2489	}
2490
2491	if ((sb->sb_flags & SB_LOCK) && !(flags & SBL_WAIT)) {
2492	return EWOULDBLOCK;
2493	}
2494	/*
2495	* We may get here from sorflush(), in which case "sb" may not
2496	* point to the real socket buffer. Use the actual socket buffer
2497	* address from the socket instead.
2498	*/
2499	wchan = (sb->sb_flags & SB_RECV) ?
2500	&so->so_rcv.sb_flags : &so->so_snd.sb_flags;
2501
2502	/*
2503	* A content filter thread has exclusive access to the sockbuf
2504	* until it clears the
2505	*/
2506	while ((sb->sb_flags & SB_LOCK) \|\|
2507	((so->so_flags & SOF_CONTENT_FILTER) &&
2508	sb->sb_cfil_thread != NULL)) {
2509	lck_mtx_t *mutex_held;
2510
2511	/*
2512	* XXX: This code should be moved up above outside of this loop;
2513	* however, we may get here as part of sofreelastref(), and
2514	* at that time pr_getlock() may no longer be able to return
2515	* us the lock. This will be fixed in future.
2516	*/
2517	if (so->so_proto->pr_getlock != NULL) {
2518	mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK);
2519	} else {
2520	mutex_held = so->so_proto->pr_domain->dom_mtx;
2521	}
2522
2523	LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
2524
2525	sb->sb_wantlock++;
2526	VERIFY(sb->sb_wantlock != `0`);
2527
2528	error = msleep(chan: wchan, mtx: mutex_held,
2529	pri: nointr ? PSOCK : PSOCK \| PCATCH,
2530	wmesg: nointr ? "sb_lock_nointr" : "sb_lock", NULL);
2531
2532	VERIFY(sb->sb_wantlock != `0`);
2533	sb->sb_wantlock--;
2534
2535	if (error == `0` && (so->so_flags & SOF_DEFUNCT) &&
2536	!(flags & SBL_IGNDEFUNCT)) {
2537	error = EBADF;
2538	SODEFUNCTLOG("%s[%d, %s]: defunct so 0x%llu [%d,%d] "
2539	"(%d)\n", __func__, proc_selfpid(),
2540	proc_best_name(current_proc()),
2541	so->so_gencnt,
2542	SOCK_DOM(so), SOCK_TYPE(so), error);
2543	}
2544
2545	if (error != `0`) {
2546	return error;
2547	}
2548	}
2549	sb->sb_flags \|= SB_LOCK;
2550	return `0`;
2551	}
2552
2553	/*
2554	* Release lock on sockbuf sb
2555	*/
2556	void
2557	sbunlock(struct sockbuf *sb, boolean_t keeplocked)
2558	{
2559	void *lr_saved = __builtin_return_address(`0`);
2560	struct socket *so = sb->sb_so;
2561	thread_t tp = current_thread();
2562
2563	/ so_usecount may be 0 if we get here from sofreelastref() /
2564	if (so == NULL) {
2565	panic("%s: null so, sb=%p sb_flags=0x%x lr=%p",
2566	__func__, sb, sb->sb_flags, lr_saved);
2567	/ NOTREACHED /
2568	} else if (so->so_usecount < `0`) {
2569	panic("%s: sb=%p sb_flags=0x%x sb_so=%p usecount=%d lr=%p "
2570	"lrh= %s\n", __func__, sb, sb->sb_flags, so,
2571	so->so_usecount, lr_saved, solockhistory_nr(so));
2572	/ NOTREACHED /
2573	}
2574
2575	/*
2576	* The content filter thread must hold the sockbuf lock
2577	*/
2578	if ((so->so_flags & SOF_CONTENT_FILTER) && sb->sb_cfil_thread == tp) {
2579	/*
2580	* Don't panic if we are defunct because SB_LOCK has
2581	* been cleared by sodefunct()
2582	*/
2583	if (!(so->so_flags & SOF_DEFUNCT) &&
2584	!(sb->sb_flags & SB_LOCK) &&
2585	!(so->so_state & SS_DEFUNCT) &&
2586	!(so->so_flags1 & SOF1_DEFUNCTINPROG)) {
2587	panic("%s: SB_LOCK not held for %p",
2588	__func__, sb);
2589	}
2590	/ Keep the sockbuf locked and proceed /
2591	} else {
2592	VERIFY((sb->sb_flags & SB_LOCK) \|\|
2593	(so->so_state & SS_DEFUNCT) \|\|
2594	(so->so_flags1 & SOF1_DEFUNCTINPROG));
2595
2596	sb->sb_flags &= ~SB_LOCK;
2597
2598	if (sb->sb_wantlock > `0`) {
2599	/*
2600	* We may get here from sorflush(), in which case "sb"
2601	* may not point to the real socket buffer. Use the
2602	* actual socket buffer address from the socket instead.
2603	*/
2604	wakeup(chan: (sb->sb_flags & SB_RECV) ? &so->so_rcv.sb_flags :
2605	&so->so_snd.sb_flags);
2606	}
2607	}
2608
2609	if (!keeplocked) { / unlock on exit /
2610	if (so->so_flags & SOF_MP_SUBFLOW \|\| SOCK_DOM(so) == PF_MULTIPATH) {
2611	(*so->so_proto->pr_unlock)(so, `1`, lr_saved);
2612	} else {
2613	lck_mtx_t *mutex_held;
2614
2615	if (so->so_proto->pr_getlock != NULL) {
2616	mutex_held = (*so->so_proto->pr_getlock)(so, PR_F_WILLUNLOCK);
2617	} else {
2618	mutex_held = so->so_proto->pr_domain->dom_mtx;
2619	}
2620
2621	LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
2622
2623	VERIFY(so->so_usecount > `0`);
2624	so->so_usecount--;
2625	so->unlock_lr[so->next_unlock_lr] = lr_saved;
2626	so->next_unlock_lr = (so->next_unlock_lr + `1`) % SO_LCKDBG_MAX;
2627	lck_mtx_unlock(lck: mutex_held);
2628	}
2629	}
2630	}
2631
2632	void
2633	sorwakeup(struct socket *so)
2634	{
2635	if (sb_notify(sb: &so->so_rcv)) {
2636	sowakeup(so, sb: &so->so_rcv, NULL);
2637	}
2638	}
2639
2640	void
2641	sowwakeup(struct socket *so)
2642	{
2643	if (sb_notify(sb: &so->so_snd)) {
2644	sowakeup(so, sb: &so->so_snd, NULL);
2645	}
2646	}
2647
2648	static void
2649	soevupcall(struct socket *so, uint32_t hint)
2650	{
2651	if (so->so_event != NULL) {
2652	caddr_t so_eventarg = so->so_eventarg;
2653
2654	hint &= so->so_eventmask;
2655	if (hint != `0`) {
2656	so->so_event(so, so_eventarg, hint);
2657	}
2658	}
2659	}
2660
2661	void
2662	soevent(struct socket *so, uint32_t hint)
2663	{
2664	if (net_wake_pkt_debug > `0` && (hint & SO_FILT_HINT_WAKE_PKT)) {
2665	os_log(OS_LOG_DEFAULT, "%s: SO_FILT_HINT_WAKE_PKT so %p",
2666	__func__, so);
2667	}
2668
2669	if (so->so_flags & SOF_KNOTE) {
2670	KNOTE(&so->so_klist, hint);
2671	}
2672
2673	soevupcall(so, hint);
2674
2675	/*
2676	* Don't post an event if this a subflow socket or
2677	* the app has opted out of using cellular interface
2678	*/
2679	if ((hint & SO_FILT_HINT_IFDENIED) &&
2680	!(so->so_flags & SOF_MP_SUBFLOW) &&
2681	!(so->so_restrictions & SO_RESTRICT_DENY_CELLULAR) &&
2682	!(so->so_restrictions & SO_RESTRICT_DENY_EXPENSIVE) &&
2683	!(so->so_restrictions & SO_RESTRICT_DENY_CONSTRAINED)) {
2684	soevent_ifdenied(so);
2685	}
2686	}
2687
2688	static void
2689	soevent_ifdenied(struct socket *so)
2690	{
2691	struct kev_netpolicy_ifdenied ev_ifdenied;
2692
2693	bzero(s: &ev_ifdenied, n: sizeof(ev_ifdenied));
2694	/*
2695	* The event consumer is interested about the effective {upid,pid,uuid}
2696	* info which can be different than the those related to the process
2697	* that recently performed a system call on the socket, i.e. when the
2698	* socket is delegated.
2699	*/
2700	if (so->so_flags & SOF_DELEGATED) {
2701	ev_ifdenied.ev_data.eupid = so->e_upid;
2702	ev_ifdenied.ev_data.epid = so->e_pid;
2703	uuid_copy(dst: ev_ifdenied.ev_data.euuid, src: so->e_uuid);
2704	} else {
2705	ev_ifdenied.ev_data.eupid = so->last_upid;
2706	ev_ifdenied.ev_data.epid = so->last_pid;
2707	uuid_copy(dst: ev_ifdenied.ev_data.euuid, src: so->last_uuid);
2708	}
2709
2710	if (++so->so_ifdenied_notifies > `1`) {
2711	/*
2712	* Allow for at most one kernel event to be generated per
2713	* socket; so_ifdenied_notifies is reset upon changes in
2714	* the UUID policy. See comments in inp_update_policy.
2715	*/
2716	if (net_io_policy_log) {
2717	uuid_string_t buf;
2718
2719	uuid_unparse(uu: ev_ifdenied.ev_data.euuid, out: buf);
2720	log(LOG_DEBUG, "%s[%d]: so %llu [%d,%d] epid %llu "
2721	"euuid %s%s has %d redundant events supressed\n",
2722	__func__, so->last_pid,
2723	so->so_gencnt, SOCK_DOM(so),
2724	SOCK_TYPE(so), ev_ifdenied.ev_data.epid, buf,
2725	((so->so_flags & SOF_DELEGATED) ?
2726	" [delegated]" : ""), so->so_ifdenied_notifies);
2727	}
2728	} else {
2729	if (net_io_policy_log) {
2730	uuid_string_t buf;
2731
2732	uuid_unparse(uu: ev_ifdenied.ev_data.euuid, out: buf);
2733	log(LOG_DEBUG, "%s[%d]: so %llu [%d,%d] epid %llu "
2734	"euuid %s%s event posted\n", __func__,
2735	so->last_pid, so->so_gencnt,
2736	SOCK_DOM(so), SOCK_TYPE(so),
2737	ev_ifdenied.ev_data.epid, buf,
2738	((so->so_flags & SOF_DELEGATED) ?
2739	" [delegated]" : ""));
2740	}
2741	netpolicy_post_msg(KEV_NETPOLICY_IFDENIED, &ev_ifdenied.ev_data,
2742	sizeof(ev_ifdenied));
2743	}
2744	}
2745
2746	/*
2747	* Make a copy of a sockaddr in a malloced buffer of type SONAME.
2748	*/
2749	struct sockaddr *
2750	dup_sockaddr(struct sockaddr sa, int* canwait)
2751	{
2752	struct sockaddr *sa2;
2753
2754	sa2 = SA(alloc_sockaddr(sa->sa_len, canwait ? Z_WAITOK : Z_NOWAIT));
2755	if (sa2 != NULL) {
2756	SOCKADDR_COPY(sa, sa2, sa->sa_len);
2757	}
2758	return sa2;
2759	}
2760
2761	void * __header_indexable
2762	alloc_sockaddr(size_t size, zalloc_flags_t flags)
2763	{
2764	VERIFY((size) <= UINT8_MAX);
2765
2766	__typed_allocators_ignore_push
2767	void * buf = kheap_alloc(KHEAP_SONAME, size, flags \| Z_ZERO);
2768	__typed_allocators_ignore_pop
2769	if (buf != NULL) {
2770	struct sockaddr *sa = SA(buf);
2771	sa->sa_len = (uint8_t)size;
2772	}
2773
2774	return buf;
2775	}
2776
2777	/*
2778	* Create an external-format (``xsocket'') structure using the information
2779	* in the kernel-format socket structure pointed to by so. This is done
2780	* to reduce the spew of irrelevant information over this interface,
2781	* to isolate user code from changes in the kernel structure, and
2782	* potentially to provide information-hiding if we decide that
2783	* some of this information should be hidden from users.
2784	*/
2785	void
2786	sotoxsocket(struct socket so, struct* xsocket *xso)
2787	{
2788	xso->xso_len = sizeof(*xso);
2789	xso->xso_so = (_XSOCKET_PTR(struct socket *))VM_KERNEL_ADDRHASH(so);
2790	xso->so_type = so->so_type;
2791	xso->so_options = (short)(so->so_options & `0xffff`);
2792	xso->so_linger = so->so_linger;
2793	xso->so_state = so->so_state;
2794	xso->so_pcb = (_XSOCKET_PTR(caddr_t))VM_KERNEL_ADDRHASH(so->so_pcb);
2795	if (so->so_proto) {
2796	xso->xso_protocol = SOCK_PROTO(so);
2797	xso->xso_family = SOCK_DOM(so);
2798	} else {
2799	xso->xso_protocol = xso->xso_family = `0`;
2800	}
2801	xso->so_qlen = so->so_qlen;
2802	xso->so_incqlen = so->so_incqlen;
2803	xso->so_qlimit = so->so_qlimit;
2804	xso->so_timeo = so->so_timeo;
2805	xso->so_error = so->so_error;
2806	xso->so_pgid = so->so_pgid;
2807	xso->so_oobmark = so->so_oobmark;
2808	sbtoxsockbuf(sb: &so->so_snd, xsb: &xso->so_snd);
2809	sbtoxsockbuf(sb: &so->so_rcv, xsb: &xso->so_rcv);
2810	xso->so_uid = kauth_cred_getuid(cred: so->so_cred);
2811	}
2812
2813
2814	#if XNU_TARGET_OS_OSX
2815
2816	void
2817	sotoxsocket64(struct socket so, struct* xsocket64 *xso)
2818	{
2819	xso->xso_len = sizeof(*xso);
2820	xso->xso_so = (u_int64_t)VM_KERNEL_ADDRHASH(so);
2821	xso->so_type = so->so_type;
2822	xso->so_options = (short)(so->so_options & `0xffff`);
2823	xso->so_linger = so->so_linger;
2824	xso->so_state = so->so_state;
2825	xso->so_pcb = (u_int64_t)VM_KERNEL_ADDRHASH(so->so_pcb);
2826	if (so->so_proto) {
2827	xso->xso_protocol = SOCK_PROTO(so);
2828	xso->xso_family = SOCK_DOM(so);
2829	} else {
2830	xso->xso_protocol = xso->xso_family = `0`;
2831	}
2832	xso->so_qlen = so->so_qlen;
2833	xso->so_incqlen = so->so_incqlen;
2834	xso->so_qlimit = so->so_qlimit;
2835	xso->so_timeo = so->so_timeo;
2836	xso->so_error = so->so_error;
2837	xso->so_pgid = so->so_pgid;
2838	xso->so_oobmark = so->so_oobmark;
2839	sbtoxsockbuf(sb: &so->so_snd, xsb: &xso->so_snd);
2840	sbtoxsockbuf(sb: &so->so_rcv, xsb: &xso->so_rcv);
2841	xso->so_uid = kauth_cred_getuid(cred: so->so_cred);
2842	}
2843
2844	#endif /* XNU_TARGET_OS_OSX */
2845
2846	/*
2847	* This does the same for sockbufs. Note that the xsockbuf structure,
2848	* since it is always embedded in a socket, does not include a self
2849	* pointer nor a length. We make this entry point public in case
2850	* some other mechanism needs it.
2851	*/
2852	void
2853	sbtoxsockbuf(struct sockbuf sb, struct* xsockbuf *xsb)
2854	{
2855	xsb->sb_cc = sb->sb_cc;
2856	xsb->sb_hiwat = sb->sb_hiwat;
2857	xsb->sb_mbcnt = sb->sb_mbcnt;
2858	xsb->sb_mbmax = sb->sb_mbmax;
2859	xsb->sb_lowat = sb->sb_lowat;
2860	xsb->sb_flags = (short)sb->sb_flags;
2861	xsb->sb_timeo = (short)
2862	((sb->sb_timeo.tv_sec * hz) + sb->sb_timeo.tv_usec / tick);
2863	if (xsb->sb_timeo == `0` && sb->sb_timeo.tv_usec != `0`) {
2864	xsb->sb_timeo = `1`;
2865	}
2866	}
2867
2868	/*
2869	* Based on the policy set by an all knowing decison maker, throttle sockets
2870	* that either have been marked as belonging to "background" process.
2871	*/
2872	inline int
2873	soisthrottled(struct socket *so)
2874	{
2875	return so->so_flags1 & SOF1_TRAFFIC_MGT_SO_BACKGROUND;
2876	}
2877
2878	inline int
2879	soisprivilegedtraffic(struct socket *so)
2880	{
2881	return (so->so_flags & SOF_PRIVILEGED_TRAFFIC_CLASS) ? `1` : `0`;
2882	}
2883
2884	inline int
2885	soissrcbackground(struct socket *so)
2886	{
2887	return (so->so_flags1 & SOF1_TRAFFIC_MGT_SO_BACKGROUND) \|\|
2888	IS_SO_TC_BACKGROUND(so->so_traffic_class);
2889	}
2890
2891	inline int
2892	soissrcrealtime(struct socket *so)
2893	{
2894	return so->so_traffic_class >= SO_TC_AV &&
2895	so->so_traffic_class <= SO_TC_VO;
2896	}
2897
2898	inline int
2899	soissrcbesteffort(struct socket *so)
2900	{
2901	return so->so_traffic_class == SO_TC_BE \|\|
2902	so->so_traffic_class == SO_TC_RD \|\|
2903	so->so_traffic_class == SO_TC_OAM;
2904	}
2905
2906	void
2907	soclearfastopen(struct socket *so)
2908	{
2909	if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2910	so->so_flags1 &= ~SOF1_PRECONNECT_DATA;
2911	}
2912
2913	if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
2914	so->so_flags1 &= ~SOF1_DATA_IDEMPOTENT;
2915	}
2916	}
2917
2918	void
2919	sonullevent(struct socket so, void* *arg, uint32_t hint)
2920	{
2921	#pragma unused(so, arg, hint)
2922	}
2923
2924	/*
2925	* Here is the definition of some of the basic objects in the kern.ipc
2926	* branch of the MIB.
2927	*/
2928	SYSCTL_NODE(_kern, KERN_IPC, ipc,
2929	CTLFLAG_RW \| CTLFLAG_LOCKED \| CTLFLAG_ANYBODY, `0`, "IPC");
2930
2931	/ Check that the maximum socket buffer size is within a range /
2932
2933	static int
2934	sysctl_sb_max SYSCTL_HANDLER_ARGS
2935	{
2936	#pragma unused(oidp, arg1, arg2)
2937	u_int32_t new_value;
2938	int changed = `0`;
2939	int error = sysctl_io_number(req, bigValue: sb_max, valueSize: sizeof(u_int32_t),
2940	pValue: &new_value, changed: &changed);
2941	if (!error && changed) {
2942	if (new_value > LOW_SB_MAX && new_value <= high_sb_max) {
2943	sb_max = new_value;
2944	} else {
2945	error = ERANGE;
2946	}
2947	}
2948	return error;
2949	}
2950
2951	SYSCTL_PROC(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf,
2952	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_LOCKED,
2953	&sb_max, `0`, &sysctl_sb_max, "IU", "Maximum socket buffer size");
2954
2955	SYSCTL_INT(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor,
2956	CTLFLAG_RW \| CTLFLAG_LOCKED, &sb_efficiency, `0`, "");
2957
2958	SYSCTL_INT(_kern_ipc, KIPC_NMBCLUSTERS, nmbclusters,
2959	CTLFLAG_RD \| CTLFLAG_LOCKED, &nmbclusters, `0`, "");
2960
2961	SYSCTL_INT(_kern_ipc, OID_AUTO, njcl,
2962	CTLFLAG_RD \| CTLFLAG_LOCKED, &njcl, `0`, "");
2963
2964	SYSCTL_INT(_kern_ipc, OID_AUTO, njclbytes,
2965	CTLFLAG_RD \| CTLFLAG_LOCKED, &njclbytes, `0`, "");
2966
2967	SYSCTL_INT(_kern_ipc, KIPC_SOQLIMITCOMPAT, soqlimitcompat,
2968	CTLFLAG_RW \| CTLFLAG_LOCKED, &soqlimitcompat, `1`,
2969	"Enable socket queue limit compatibility");
2970
2971	/*
2972	* Hack alert -- rdar://33572856
2973	* A loopback test we cannot change was failing because it sets
2974	* SO_SENDTIMEO to 5 seconds and that's also the value
2975	* of the minimum persist timer. Because of the persist timer,
2976	* the connection was not idle for 5 seconds and SO_SNDTIMEO
2977	* was not triggering at 5 seconds causing the test failure.
2978	* As a workaround we check the sysctl soqlencomp the test is already
2979	* setting to set disable auto tuning of the receive buffer.
2980	*/
2981
2982	extern u_int32_t tcp_do_autorcvbuf;
2983
2984	static int
2985	sysctl_soqlencomp SYSCTL_HANDLER_ARGS
2986	{
2987	#pragma unused(oidp, arg1, arg2)
2988	u_int32_t new_value;
2989	int changed = `0`;
2990	int error = sysctl_io_number(req, bigValue: soqlencomp, valueSize: sizeof(u_int32_t),
2991	pValue: &new_value, changed: &changed);
2992	if (!error && changed) {
2993	soqlencomp = new_value;
2994	if (new_value != `0`) {
2995	tcp_do_autorcvbuf = `0`;
2996	tcptv_persmin_val = `6` * TCP_RETRANSHZ;
2997	}
2998	}
2999	return error;
3000	}
3001	SYSCTL_PROC(_kern_ipc, OID_AUTO, soqlencomp,
3002	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_LOCKED,
3003	&soqlencomp, `0`, &sysctl_soqlencomp, "IU", "");
3004
3005	SYSCTL_NODE(_kern_ipc, OID_AUTO, io_policy, CTLFLAG_RW, `0`, "network IO policy");
3006
3007	SYSCTL_INT(_kern_ipc_io_policy, OID_AUTO, log, CTLFLAG_RW \| CTLFLAG_LOCKED,
3008	&net_io_policy_log, `0`, "");
3009
3010	#if CONFIG_PROC_UUID_POLICY
3011	SYSCTL_INT(_kern_ipc_io_policy, OID_AUTO, uuid, CTLFLAG_RW \| CTLFLAG_LOCKED,
3012	&net_io_policy_uuid, `0`, "");
3013	#endif /* CONFIG_PROC_UUID_POLICY */
3014

Browse the source code of xnu/bsd/kern/uipc_socket2.c