1/*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 */
4
5#pragma once
6
7#ifdef KERNEL_PRIVATE
8#ifdef __cplusplus
9
10#include <IOKit/IOService.h>
11#include <stdatomic.h>
12#include <kern/bits.h>
13#include <libkern/c++/OSPtr.h>
14
15struct thread_group;
16
17enum{
18 kIOPerfControlClientWorkUntracked = 0,
19};
20
21/*!
22 * @class IOPerfControlClient : public OSObject
23 * @abstract Class which implements an interface allowing device drivers to participate in performance control.
24 * @discussion TODO
25 */
26class IOPerfControlClient final : public OSObject
27{
28 OSDeclareDefaultStructors(IOPerfControlClient);
29
30protected:
31 virtual bool init(IOService *driver, uint64_t maxWorkCapacity);
32 virtual void free() APPLE_KEXT_OVERRIDE;
33
34public:
35/*!
36 * @function copyClient
37 * @abstract Return a retained reference to a client object, to be released by the driver. It may be
38 * shared with other drivers in the system.
39 * @param driver The device driver that will be using this interface.
40 * @param maxWorkCapacity The maximum number of concurrent work items supported by the device driver.
41 * @returns An instance of IOPerfControlClient.
42 */
43 static IOPerfControlClient *copyClient(IOService *driver, uint64_t maxWorkCapacity);
44
45/*!
46 * @function registerDevice
47 * @abstract Inform the system that work will be dispatched to a device in the future.
48 * @discussion The system will do some one-time setup work associated with the device, and may block the
49 * current thread during the setup. Devices should not be passed to work workSubmit, workSubmitAndBegin,
50 * workBegin, or workEnd until they have been successfully registered. The unregistration process happens
51 * automatically when the device object is deallocated.
52 * @param device The device object. Some platforms require device to be a specific subclass of IOService.
53 * @returns kIOReturnSuccess or an IOReturn error code
54 */
55 virtual IOReturn registerDevice(IOService *driver, IOService *device);
56
57/*!
58 * @function unregisterDevice
59 * @abstract Inform the system that work will be no longer be dispatched to a device in the future.
60 * @discussion This call is optional as the unregistration process happens automatically when the device
61 * object is deallocated. This call may block the current thread and/or acquire locks. It should not be
62 * called until after all submitted work has been ended using workEnd.
63 * @param device The device object. Some platforms require device to be a specific subclass of IOService.
64 */
65 virtual void unregisterDevice(IOService *driver, IOService *device);
66
67/*!
68 * @struct WorkSubmitArgs
69 * @discussion Drivers may submit additional device-specific arguments related to the submission of a work item
70 * by passing a struct with WorkSubmitArgs as its first member. Note: Drivers are responsible for publishing
71 * a header file describing these arguments.
72 */
73 struct WorkSubmitArgs {
74 uint32_t version;
75 uint32_t size;
76 uint64_t submit_time;
77 uint64_t reserved[4];
78 void *driver_data;
79 };
80
81/*!
82 * @function workSubmit
83 * @abstract Tell the performance controller that work was submitted.
84 * @param device The device that will execute the work. Some platforms require device to be a
85 * specific subclass of IOService.
86 * @param args Optional device-specific arguments related to the submission of this work item.
87 * @returns A token representing this work item, which must be passed to workEnd when the work is finished
88 * unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks
89 * and a degradation of system performance.
90 */
91 virtual uint64_t workSubmit(IOService *device, WorkSubmitArgs *args = nullptr);
92
93/*!
94 * @struct WorkBeginArgs
95 * @discussion Drivers may submit additional device-specific arguments related to the start of a work item
96 * by passing a struct with WorkBeginArgs as its first member. Note: Drivers are responsible for publishing
97 * a header file describing these arguments.
98 */
99 struct WorkBeginArgs {
100 uint32_t version;
101 uint32_t size;
102 uint64_t begin_time;
103 uint64_t reserved[4];
104 void *driver_data;
105 };
106
107/*!
108 * @function workSubmitAndBegin
109 * @abstract Tell the performance controller that work was submitted and immediately began executing.
110 * @param device The device that is executing the work. Some platforms require device to be a
111 * specific subclass of IOService.
112 * @param submitArgs Optional device-specific arguments related to the submission of this work item.
113 * @param beginArgs Optional device-specific arguments related to the start of this work item.
114 * @returns A token representing this work item, which must be passed to workEnd when the work is finished
115 * unless the token equals kIOPerfControlClientWorkUntracked. Failure to do this will result in memory leaks
116 * and a degradation of system performance.
117 */
118 virtual uint64_t workSubmitAndBegin(IOService *device, WorkSubmitArgs *submitArgs = nullptr,
119 WorkBeginArgs *beginArgs = nullptr);
120
121/*!
122 * @function workBegin
123 * @abstract Tell the performance controller that previously submitted work began executing.
124 * @param device The device that is executing the work. Some platforms require device to be a
125 * specific subclass of IOService.
126 * @param args Optional device-specific arguments related to the start of this work item.
127 */
128 virtual void workBegin(IOService *device, uint64_t token, WorkBeginArgs *args = nullptr);
129
130/*!
131 * @struct WorkEndArgs
132 * @discussion Drivers may submit additional device-specific arguments related to the end of a work item
133 * by passing a struct with WorkEndArgs as its first member. Note: Drivers are responsible for publishing
134 * a header file describing these arguments.
135 */
136 struct WorkEndArgs {
137 uint32_t version;
138 uint32_t size;
139 uint64_t end_time;
140 uint64_t reserved[4];
141 void *driver_data;
142 };
143
144/*!
145 * @function workEnd
146 * @abstract Tell the performance controller that previously started work finished executing.
147 * @param device The device that executed the work. Some platforms require device to be a
148 * specific subclass of IOService.
149 * @param args Optional device-specific arguments related to the end of this work item.
150 * @param done Optional Set to false if the work has not yet completed. Drivers are then responsible for
151 * calling workBegin when the work resumes and workEnd with done set to True when it has completed. A workEnd() call
152 * without a corresponding workBegin() call is a way to cancel a work item and return token to IOPerfControl.
153 */
154 virtual void workEnd(IOService *device, uint64_t token, WorkEndArgs *args = nullptr, bool done = true);
155
156/*!
157 * @function copyWorkContext
158 * @abstract Return a retained reference to an opaque OSObject, to be released by the driver. This object can
159 * be used by IOPerfControl to track a work item. This may perform dynamic memory allocation.
160 * @returns A pointer to an OSObject
161 */
162 OSPtr<OSObject> copyWorkContext();
163
164/*!
165 * @function workSubmitAndBeginWithContext
166 * @abstract Tell the performance controller that work was submitted and immediately began executing
167 * @param device The device that is executing the work. Some platforms require device to be a
168 * specific subclass of IOService.
169 * @param context An OSObject returned by copyWorkContext(). The context object will be used by IOPerfControl to track
170 * this work item.
171 * @param submitArgs Optional device-specific arguments related to the submission of this work item.
172 * @param beginArgs Optional device-specific arguments related to the start of this work item.
173 * @returns true if IOPerfControl is tracking this work item, else false.
174 * @note The workEndWithContext() call is optional if the corresponding workSubmitWithContext() call returned false.
175 */
176 bool workSubmitAndBeginWithContext(IOService *device, OSObject *context, WorkSubmitArgs *submitArgs = nullptr,
177 WorkBeginArgs *beginArgs = nullptr);
178
179/*!
180 * @function workSubmitWithContext
181 * @abstract Tell the performance controller that work was submitted.
182 * @param device The device that will execute the work. Some platforms require device to be a
183 * specific subclass of IOService.
184 * @param context An OSObject returned by copyWorkContext(). The context object will be used by IOPerfControl to track
185 * this work item.
186 * @param args Optional device-specific arguments related to the submission of this work item.
187 * @returns true if IOPerfControl is tracking this work item, else false.
188 */
189 bool workSubmitWithContext(IOService *device, OSObject *context, WorkSubmitArgs *args = nullptr);
190
191/*!
192 * @function workBeginWithContext
193 * @abstract Tell the performance controller that previously submitted work began executing.
194 * @param device The device that is executing the work. Some platforms require device to be a
195 * specific subclass of IOService.
196 * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext().
197 * @param args Optional device-specific arguments related to the start of this work item.
198 * @note The workBeginWithContext() and workEndWithContext() calls are optional if the corresponding workSubmitWithContext() call returned false.
199 */
200 void workBeginWithContext(IOService *device, OSObject *context, WorkBeginArgs *args = nullptr);
201
202/*!
203 * @function workEndWithContext
204 * @abstract Tell the performance controller that previously started work finished executing.
205 * @param device The device that executed the work. Some platforms require device to be a
206 * specific subclass of IOService.
207 * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext().
208 * @param args Optional device-specific arguments related to the end of this work item.
209 * @param done Optional Set to false if the work has not yet completed. Drivers are then responsible for
210 * calling workBegin when the work resumes and workEnd with done set to True when it has completed.
211 * @note The workEndWithContext() call is optional if the corresponding workSubmitWithContext() call returned false. A workEndWithContext()
212 * call without a corresponding workBeginWithContext() call is a way to cancel a work item.
213 */
214 void workEndWithContext(IOService *device, OSObject *context, WorkEndArgs *args = nullptr, bool done = true);
215
216/*!
217 * @struct WorkUpdateArgs
218 * @discussion Drivers may submit additional device-specific arguments related to a work item by passing a
219 * struct with WorkUpdateArgs as its first member. Note: Drivers are responsible for publishing
220 * a header file describing these arguments.
221 */
222 struct WorkUpdateArgs {
223 uint32_t version;
224 uint32_t size;
225 uint64_t update_time;
226 uint64_t reserved[4];
227 void *driver_data;
228 };
229
230/*!
231 * @function workUpdateWithContext
232 * @abstract Provide and receive additional information from the performance controller. If this call is
233 * made at all, it should be between workSubmit and workEnd. The purpose and implementation of this call are
234 * device specific, and may do nothing on some devices.
235 * @param device The device that submitted the work. Some platforms require device to be a
236 * specific subclass of IOService.
237 * @param context An OSObject returned by copyWorkContext() and provided to the previous call to workSubmitWithContext().
238 * @param args Optional device-specific arguments.
239 */
240 void workUpdateWithContext(IOService *device, OSObject *context, WorkUpdateArgs *args = nullptr);
241
242/*
243 * Callers should always use the CURRENT version so that the kernel can detect both older
244 * and newer structure layouts. New callbacks should always be added at the end of the
245 * structure, and xnu should expect existing source recompiled against newer headers
246 * to pass NULL for unimplemented callbacks.
247 */
248
249#define PERFCONTROL_INTERFACE_VERSION_NONE (0) /* no interface */
250#define PERFCONTROL_INTERFACE_VERSION_1 (1) /* up-to workEnd */
251#define PERFCONTROL_INTERFACE_VERSION_2 (2) /* up-to workUpdate */
252#define PERFCONTROL_INTERFACE_VERSION_3 (3) /* up-to (un)registerDriverDevice */
253#define PERFCONTROL_INTERFACE_VERSION_CURRENT PERFCONTROL_INTERFACE_VERSION_3
254
255/*!
256 * @struct PerfControllerInterface
257 * @discussion Function pointers necessary to register a performance controller. Not for general driver use.
258 */
259 struct PerfControllerInterface {
260 struct DriverState {
261 uint32_t has_target_thread_group : 1;
262 uint32_t has_device_info : 1;
263 uint32_t reserved : 30;
264
265 uint64_t target_thread_group_id;
266 void *target_thread_group_data;
267
268 uint32_t device_type;
269 uint32_t instance_id;
270 };
271
272 struct WorkState {
273 uint64_t thread_group_id;
274 void *thread_group_data;
275 void *work_data;
276 uint32_t work_data_size;
277 uint32_t started : 1;
278 uint32_t reserved : 31;
279 const DriverState* driver_state;
280 };
281
282 using RegisterDeviceFunction = IOReturn (*)(IOService *);
283 using RegisterDriverDeviceFunction = IOReturn (*)(IOService *, IOService *, DriverState *);
284 using WorkCanSubmitFunction = bool (*)(IOService *, WorkState *, WorkSubmitArgs *);
285 using WorkSubmitFunction = void (*)(IOService *, uint64_t, WorkState *, WorkSubmitArgs *);
286 using WorkBeginFunction = void (*)(IOService *, uint64_t, WorkState *, WorkBeginArgs *);
287 using WorkEndFunction = void (*)(IOService *, uint64_t, WorkState *, WorkEndArgs *, bool);
288 using WorkUpdateFunction = void (*)(IOService *, uint64_t, WorkState *, WorkUpdateArgs *);
289
290 uint64_t version;
291 RegisterDeviceFunction registerDevice;
292 RegisterDeviceFunction unregisterDevice;
293 WorkCanSubmitFunction workCanSubmit;
294 WorkSubmitFunction workSubmit;
295 WorkBeginFunction workBegin;
296 WorkEndFunction workEnd;
297 WorkUpdateFunction workUpdate;
298 RegisterDriverDeviceFunction registerDriverDevice;
299 RegisterDriverDeviceFunction unregisterDriverDevice;
300 };
301
302 struct IOPerfControlClientShared {
303 atomic_uint_fast8_t maxDriverIndex;
304 PerfControllerInterface interface;
305 IOLock *interfaceLock;
306 OSSet *deviceRegistrationList;
307 };
308
309 struct IOPerfControlClientData {
310 struct thread_group *target_thread_group;
311 PerfControllerInterface::DriverState driverState;
312 IOService* device;
313 };
314/*!
315 * @function registerPerformanceController
316 * @abstract Register a performance controller to receive callbacks. Not for general driver use.
317 * @param interface Struct containing callback functions implemented by the performance controller.
318 * @returns kIOReturnSuccess or kIOReturnError if the interface was already registered.
319 */
320 virtual IOReturn registerPerformanceController(PerfControllerInterface *interface);
321
322/*!
323 * @function getClientData
324 * @abstract Not for general driver use. Only used by registerPerformanceController(). Allows performanceController to register existing IOPerfControlClient.
325 * @returns IOPerfControlData associated with a IOPerfControlClient
326 */
327 IOPerfControlClientData *
328 getClientData()
329 {
330 return &clientData;
331 }
332
333private:
334 struct WorkTableEntry {
335 struct thread_group *thread_group;
336 bool started;
337 uint8_t perfcontrol_data[32];
338 };
339
340 static constexpr size_t kMaxWorkTableNumEntries = 1024;
341 static constexpr size_t kWorkTableIndexBits = 24;
342 static constexpr size_t kWorkTableMaxSize = (1 << kWorkTableIndexBits) - 1; // - 1 since
343 // kIOPerfControlClientWorkUntracked takes number 0
344 static constexpr size_t kWorkTableIndexMask = (const size_t)mask(kWorkTableIndexBits);
345
346 uint64_t allocateToken(thread_group *thread_group);
347 void deallocateToken(uint64_t token);
348 WorkTableEntry *getEntryForToken(uint64_t token);
349 void markEntryStarted(uint64_t token, bool started);
350 inline uint64_t tokenToGlobalUniqueToken(uint64_t token);
351
352 uint8_t driverIndex;
353 IOPerfControlClientShared *shared;
354 WorkTableEntry *workTable;
355 size_t workTableLength;
356 size_t workTableNextIndex;
357 IOSimpleLock *workTableLock;
358
359 IOPerfControlClientData clientData;
360};
361
362#endif /* __cplusplus */
363#endif /* KERNEL_PRIVATE */
364