1/*
2 * Copyright (c) 2019-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <skywalk/os_skywalk.h>
30#include <skywalk/os_skywalk_private.h>
31#include <skywalk/namespace/protons.h>
32
33#include <kern/bits.h>
34#include <netinet/in_var.h>
35#include <netinet6/in6_var.h>
36#include <sys/domain.h>
37
38static int __protons_inited = 0;
39
40decl_lck_mtx_data(static, protons_lock);
41static LCK_GRP_DECLARE(protons_lock_group, "protons_lock");
42static LCK_MTX_DECLARE(protons_lock, &protons_lock_group);
43
44#define PROTONS_LOCK() \
45 lck_mtx_lock(&protons_lock)
46#define PROTONS_UNLOCK() \
47 lck_mtx_unlock(&protons_lock)
48#define PROTONS_LOCK_ASSERT_HELD() \
49 LCK_MTX_ASSERT(&protons_lock, LCK_MTX_ASSERT_OWNED)
50#define PROTONS_LOCK_ASSERT_NOTHELD() \
51 LCK_MTX_ASSERT(&protons_lock, LCK_MTX_ASSERT_NOTOWNED)
52
53os_refgrp_decl(static, protons_token_refgrp, "protons_token", NULL);
54
55struct protons_token {
56 RB_ENTRY(protons_token) pt_link;
57 os_refcnt_t pt_refcnt;
58 pid_t pt_pid;
59 pid_t pt_epid;
60 uint8_t pt_protocol;
61 uint8_t pt_flags;
62};
63
64enum {
65 PROTONSF_VALID = (((uint8_t)1) << 0),
66};
67
68__attribute__((always_inline))
69static inline int
70pt_cmp(const struct protons_token *pt1, const struct protons_token *pt2)
71{
72 return (int)pt1->pt_protocol - (int)pt2->pt_protocol;
73}
74RB_HEAD(protons_token_tree, protons_token);
75RB_PROTOTYPE_PREV(protons_token_tree, protons_token, pt_link, pt_cmp);
76RB_GENERATE_PREV(protons_token_tree, protons_token, pt_link, pt_cmp);
77static struct protons_token_tree protons_tokens;
78
79static SKMEM_TYPE_DEFINE(protons_token_zone, struct protons_token);
80
81static struct protons_token *
82protons_token_alloc(bool can_block)
83{
84 PROTONS_LOCK_ASSERT_HELD();
85
86 struct protons_token *pt = NULL;
87 pt = can_block ? zalloc(kt_view: protons_token_zone) :
88 zalloc_noblock(kt_view: protons_token_zone);
89 if (pt == NULL) {
90 return NULL;
91 }
92
93 memset(s: pt, c: 0, n: sizeof(*pt));
94 os_ref_init(&pt->pt_refcnt, &protons_token_refgrp);
95
96 SK_DF(SK_VERB_NS_PROTO, "token %p alloc", (void *)SK_KVA(pt));
97
98 return pt;
99}
100
101static void
102protons_token_free(struct protons_token *pt)
103{
104 PROTONS_LOCK_ASSERT_HELD();
105
106 SK_DF(SK_VERB_NS_PROTO, "token %p free", (void *)SK_KVA(pt));
107 ASSERT(os_ref_get_count(&pt->pt_refcnt) == 0);
108 zfree(protons_token_zone, pt);
109}
110
111bool
112protons_token_is_valid(struct protons_token *pt)
113{
114 if (__improbable(pt == NULL)) {
115 return false;
116 }
117 return pt->pt_flags & PROTONSF_VALID;
118}
119
120bool
121protons_token_has_matching_pid(struct protons_token *pt, pid_t pid, pid_t epid)
122{
123 ASSERT(pt != NULL);
124 return pt->pt_pid == pid && pt->pt_epid == epid;
125}
126
127static struct protons_token *
128protons_find_token_with_protocol(uint8_t proto)
129{
130 struct protons_token find = { .pt_protocol = proto };
131
132 PROTONS_LOCK_ASSERT_HELD();
133 struct protons_token *pt = NULL;
134 pt = RB_FIND(protons_token_tree, &protons_tokens, &find);
135 if (pt) {
136 os_ref_retain(rc: &pt->pt_refcnt);
137 }
138 return pt;
139}
140
141int
142protons_token_get_use_count(struct protons_token *pt)
143{
144 /* minus one refcnt in RB tree*/
145 return os_ref_get_count(rc: &pt->pt_refcnt) - 1;
146}
147
148static void
149protons_token_release(struct protons_token *pt)
150{
151 os_ref_count_t refcnt = os_ref_release(rc: &pt->pt_refcnt);
152
153 SK_DF(SK_VERB_NS_PROTO,
154 "token %p proto %u released by pid %d epid %d, curr use %u",
155 (void *)SK_KVA(pt), pt->pt_protocol, pt->pt_pid, pt->pt_epid,
156 protons_token_get_use_count(pt));
157
158 if (refcnt == 1) {
159 RB_REMOVE(protons_token_tree, &protons_tokens, pt);
160 (void) os_ref_release(rc: &pt->pt_refcnt);
161 pt->pt_flags &= ~PROTONSF_VALID;
162 pt->pt_protocol = 0;
163 pt->pt_pid = 0;
164 pt->pt_epid = 0;
165 protons_token_free(pt);
166 }
167}
168
169static int
170protons_reserve_locked(struct protons_token **ptp, pid_t pid, pid_t epid,
171 uint8_t proto)
172{
173 struct protons_token *pt = NULL, *dup = NULL;
174 *ptp = NULL;
175
176 pt = protons_find_token_with_protocol(proto);
177 if (pt != NULL) {
178 /* use previously reserved token with same process */
179 ASSERT(pt->pt_flags & PROTONSF_VALID);
180 if (pt->pt_pid != pid || pt->pt_epid != epid) {
181 SK_ERR("proto %u existed with pid %d epid %d",
182 proto, pt->pt_pid, pt->pt_epid);
183 (void) os_ref_release(rc: &pt->pt_refcnt);
184 pt = NULL;
185 return EEXIST;
186 }
187 } else {
188 /* start with new token */
189 pt = protons_token_alloc(true);
190 if (pt == NULL) {
191 return ENOMEM;
192 }
193
194 os_ref_retain(rc: &pt->pt_refcnt);
195 pt->pt_flags |= PROTONSF_VALID;
196 pt->pt_pid = pid;
197 pt->pt_epid = (epid != -1) ? epid : pid;
198 pt->pt_protocol = proto;
199 dup = RB_INSERT(protons_token_tree, &protons_tokens, pt);
200 ASSERT(dup == NULL);
201 }
202
203 SK_DF(SK_VERB_NS_PROTO,
204 "token %p proto %u reserved by pid %d epid %d, curr use %u",
205 (void *)SK_KVA(pt), proto, pid, epid, protons_token_get_use_count(pt));
206 *ptp = pt;
207
208 return 0;
209}
210
211int
212protons_reserve(struct protons_token **ptp, pid_t pid, pid_t epid,
213 uint8_t proto)
214{
215 int err = 0;
216 PROTONS_LOCK();
217 err = protons_reserve_locked(ptp, pid, epid, proto);
218 PROTONS_UNLOCK();
219 return err;
220}
221
222void
223protons_release(struct protons_token **ptp)
224{
225 struct protons_token *pt = *ptp;
226 ASSERT(pt != NULL);
227
228 PROTONS_LOCK();
229 protons_token_release(pt);
230 PROTONS_UNLOCK();
231 *ptp = NULL;
232}
233
234/* Reserved all protocol used by BSD stack. */
235static void
236protons_init_netinet_protocol(void)
237{
238 PROTONS_LOCK();
239
240 uint8_t proto = 0;
241 struct protons_token *pt = NULL;
242 int error = 0;
243 struct protosw *pp = NULL;
244 TAILQ_FOREACH(pp, &inetdomain->dom_protosw, pr_entry) {
245 pt = NULL;
246 proto = (uint8_t)pp->pr_protocol;
247 error = protons_reserve_locked(ptp: &pt, pid: 0, epid: 0, proto);
248 VERIFY(error == 0 || error == EEXIST);
249 VERIFY(pt != NULL);
250 }
251
252 TAILQ_FOREACH(pp, &inet6domain->dom_protosw, pr_entry) {
253 pt = NULL;
254 proto = (uint8_t)pp->pr_protocol;
255 error = protons_reserve_locked(ptp: &pt, pid: 0, epid: 0, proto);
256 VERIFY(error == 0 || error == EEXIST);
257 VERIFY(pt != NULL);
258 }
259
260 PROTONS_UNLOCK();
261}
262
263int
264protons_init(void)
265{
266 VERIFY(__protons_inited == 0);
267
268 RB_INIT(&protons_tokens);
269
270 protons_init_netinet_protocol();
271
272 __protons_inited = 1;
273 sk_features |= SK_FEATURE_PROTONS;
274
275 SK_D("initialized protons");
276
277 return 0;
278}
279
280void
281protons_fini(void)
282{
283 if (__protons_inited == 1) {
284 __protons_inited = 0;
285 sk_features &= ~SK_FEATURE_PROTONS;
286 }
287}
288
289static int protons_stats_sysctl SYSCTL_HANDLER_ARGS;
290SYSCTL_PROC(_kern_skywalk_stats, OID_AUTO, protons,
291 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
292 0, 0, protons_stats_sysctl, "", "");
293
294static int
295protons_stats_sysctl SYSCTL_HANDLER_ARGS
296{
297#pragma unused(arg1, arg2, oidp)
298 int error = 0;
299 size_t actual_space;
300 caddr_t buffer = NULL;
301 size_t buffer_space;
302 size_t allocated_space;
303 int out_error;
304 caddr_t scan;
305
306 if (!kauth_cred_issuser(cred: kauth_cred_get())) {
307 return EPERM;
308 }
309
310 net_update_uptime();
311 buffer_space = req->oldlen;
312 if (req->oldptr != USER_ADDR_NULL && buffer_space != 0) {
313 if (buffer_space > SK_SYSCTL_ALLOC_MAX) {
314 buffer_space = SK_SYSCTL_ALLOC_MAX;
315 }
316 allocated_space = buffer_space;
317 buffer = sk_alloc_data(allocated_space, Z_WAITOK, skmem_tag_sysctl_buf);
318 if (__improbable(buffer == NULL)) {
319 return ENOBUFS;
320 }
321 } else if (req->oldptr == USER_ADDR_NULL) {
322 buffer_space = 0;
323 }
324 actual_space = 0;
325 scan = buffer;
326
327 struct sk_stats_protons_token *spt = (void *)scan;
328 size_t spt_size = sizeof(*spt);
329 struct protons_token *pt = NULL;
330 PROTONS_LOCK();
331 RB_FOREACH(pt, protons_token_tree, &protons_tokens) {
332 if (scan != NULL) {
333 if (buffer_space < spt_size) {
334 /* supplied buffer too small, stop copying */
335 error = ENOMEM;
336 break;
337 }
338 spt->spt_protocol = pt->pt_protocol;
339 spt->spt_refcnt = protons_token_get_use_count(pt);
340 spt->spt_pid = pt->pt_pid;
341 spt->spt_epid = pt->pt_epid;
342 spt++;
343 buffer_space -= spt_size;
344 }
345 actual_space += spt_size;
346 }
347 PROTONS_UNLOCK();
348
349 if (actual_space != 0) {
350 out_error = SYSCTL_OUT(req, buffer, actual_space);
351 if (out_error != 0) {
352 error = out_error;
353 }
354 }
355 if (buffer != NULL) {
356 sk_free_data(buffer, allocated_space);
357 }
358 return error;
359}
360