1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Gasket generic driver. Defines the set of data types and functions necessary
4 * to define a driver using the Gasket generic driver framework.
5 *
6 * Copyright (C) 2018 Google, Inc.
7 */
8 #ifndef __GASKET_CORE_H__
9 #define __GASKET_CORE_H__
10
11 #include <linux/cdev.h>
12 #include <linux/compiler.h>
13 #include <linux/device.h>
14 #include <linux/init.h>
15 #include <linux/module.h>
16 #include <linux/pci.h>
17 #include <linux/sched.h>
18 #include <linux/slab.h>
19
20 #include "gasket_constants.h"
21
22 /**
23 * struct gasket_num_name - Map numbers to names.
24 * @ein_num: Number.
25 * @ein_name: Name associated with the number, a char pointer.
26 *
27 * This structure maps numbers to names. It is used to provide printable enum
28 * names, e.g {0, "DEAD"} or {1, "ALIVE"}.
29 */
30 struct gasket_num_name {
31 uint snn_num;
32 const char *snn_name;
33 };
34
35 /*
36 * Register location for packed interrupts.
37 * Each value indicates the location of an interrupt field (in units of
38 * gasket_driver_desc->interrupt_pack_width) within the containing register.
39 * In other words, this indicates the shift to use when creating a mask to
40 * extract/set bits within a register for a given interrupt.
41 */
42 enum gasket_interrupt_packing {
43 PACK_0 = 0,
44 PACK_1 = 1,
45 PACK_2 = 2,
46 PACK_3 = 3,
47 UNPACKED = 4,
48 };
49
50 /* Type of the interrupt supported by the device. */
51 enum gasket_interrupt_type {
52 PCI_MSIX = 0,
53 };
54
55 /*
56 * Used to describe a Gasket interrupt. Contains an interrupt index, a register,
57 * and packing data for that interrupt. The register and packing data
58 * fields are relevant only for PCI_MSIX interrupt type and can be
59 * set to 0 for everything else.
60 */
61 struct gasket_interrupt_desc {
62 /* Device-wide interrupt index/number. */
63 int index;
64 /* The register offset controlling this interrupt. */
65 u64 reg;
66 /* The location of this interrupt inside register reg, if packed. */
67 int packing;
68 };
69
70 /*
71 * This enum is used to identify memory regions being part of the physical
72 * memory that belongs to a device.
73 */
74 enum mappable_area_type {
75 PCI_BAR = 0, /* Default */
76 BUS_REGION, /* For SYSBUS devices, i.e. AXI etc... */
77 COHERENT_MEMORY
78 };
79
80 /*
81 * Metadata for each BAR mapping.
82 * This struct is used so as to track PCI memory, I/O space, AXI and coherent
83 * memory area... i.e. memory objects which can be referenced in the device's
84 * mmap function.
85 */
86 struct gasket_bar_data {
87 /* Virtual base address. */
88 u8 __iomem *virt_base;
89
90 /* Physical base address. */
91 ulong phys_base;
92
93 /* Length of the mapping. */
94 ulong length_bytes;
95
96 /* Type of mappable area */
97 enum mappable_area_type type;
98 };
99
100 /* Maintains device open ownership data. */
101 struct gasket_ownership {
102 /* 1 if the device is owned, 0 otherwise. */
103 int is_owned;
104
105 /* TGID of the owner. */
106 pid_t owner;
107
108 /* Count of current device opens in write mode. */
109 int write_open_count;
110 };
111
112 /* Page table modes of operation. */
113 enum gasket_page_table_mode {
114 /* The page table is partitionable as normal, all simple by default. */
115 GASKET_PAGE_TABLE_MODE_NORMAL,
116
117 /* All entries are always simple. */
118 GASKET_PAGE_TABLE_MODE_SIMPLE,
119
120 /* All entries are always extended. No extended bit is used. */
121 GASKET_PAGE_TABLE_MODE_EXTENDED,
122 };
123
124 /* Page table configuration. One per table. */
125 struct gasket_page_table_config {
126 /* The identifier/index of this page table. */
127 int id;
128
129 /* The operation mode of this page table. */
130 enum gasket_page_table_mode mode;
131
132 /* Total (first-level) entries in this page table. */
133 ulong total_entries;
134
135 /* Base register for the page table. */
136 int base_reg;
137
138 /*
139 * Register containing the extended page table. This value is unused in
140 * GASKET_PAGE_TABLE_MODE_SIMPLE and GASKET_PAGE_TABLE_MODE_EXTENDED
141 * modes.
142 */
143 int extended_reg;
144
145 /* The bit index indicating whether a PT entry is extended. */
146 int extended_bit;
147 };
148
149 /* Maintains information about a device node. */
150 struct gasket_cdev_info {
151 /* The internal name of this device. */
152 char name[GASKET_NAME_MAX];
153
154 /* Device number. */
155 dev_t devt;
156
157 /* Kernel-internal device structure. */
158 struct device *device;
159
160 /* Character device for real. */
161 struct cdev cdev;
162
163 /* Flag indicating if cdev_add has been called for the devices. */
164 int cdev_added;
165
166 /* Pointer to the overall gasket_dev struct for this device. */
167 struct gasket_dev *gasket_dev_ptr;
168
169 /* Ownership data for the device in question. */
170 struct gasket_ownership ownership;
171 };
172
173 /* Describes the offset and length of mmapable device BAR regions. */
174 struct gasket_mappable_region {
175 u64 start;
176 u64 length_bytes;
177 };
178
179 /* Describe the offset, size, and permissions for a device bar. */
180 struct gasket_bar_desc {
181 /*
182 * The size of each PCI BAR range, in bytes. If a value is 0, that BAR
183 * will not be mapped into kernel space at all.
184 * For devices with 64 bit BARs, only elements 0, 2, and 4 should be
185 * populated, and 1, 3, and 5 should be set to 0.
186 * For example, for a device mapping 1M in each of the first two 64-bit
187 * BARs, this field would be set as { 0x100000, 0, 0x100000, 0, 0, 0 }
188 * (one number per bar_desc struct.)
189 */
190 u64 size;
191 /* The permissions for this bar. (Should be VM_WRITE/VM_READ/VM_EXEC,
192 * and can be or'd.) If set to GASKET_NOMAP, the bar will
193 * not be used for mmapping.
194 */
195 ulong permissions;
196 /* The memory address corresponding to the base of this bar, if used. */
197 u64 base;
198 /* The number of mappable regions in this bar. */
199 int num_mappable_regions;
200
201 /* The mappable subregions of this bar. */
202 const struct gasket_mappable_region *mappable_regions;
203
204 /* Type of mappable area */
205 enum mappable_area_type type;
206 };
207
208 /* Describes the offset, size, and permissions for a coherent buffer. */
209 struct gasket_coherent_buffer_desc {
210 /* The size of the coherent buffer. */
211 u64 size;
212
213 /* The permissions for this bar. (Should be VM_WRITE/VM_READ/VM_EXEC,
214 * and can be or'd.) If set to GASKET_NOMAP, the bar will
215 * not be used for mmaping.
216 */
217 ulong permissions;
218
219 /* device side address. */
220 u64 base;
221 };
222
223 /* Coherent buffer structure. */
224 struct gasket_coherent_buffer {
225 /* Virtual base address. */
226 u8 *virt_base;
227
228 /* Physical base address. */
229 ulong phys_base;
230
231 /* Length of the mapping. */
232 ulong length_bytes;
233 };
234
235 /* Description of Gasket-specific permissions in the mmap field. */
236 enum gasket_mapping_options { GASKET_NOMAP = 0 };
237
238 /* This struct represents an undefined bar that should never be mapped. */
239 #define GASKET_UNUSED_BAR \
240 { \
241 0, GASKET_NOMAP, 0, 0, NULL, 0 \
242 }
243
244 /* Internal data for a Gasket device. See gasket_core.c for more information. */
245 struct gasket_internal_desc;
246
247 #define MAX_NUM_COHERENT_PAGES 16
248
249 /*
250 * Device data for Gasket device instances.
251 *
252 * This structure contains the data required to manage a Gasket device.
253 */
254 struct gasket_dev {
255 /* Pointer to the internal driver description for this device. */
256 struct gasket_internal_desc *internal_desc;
257
258 /* Device info */
259 struct device *dev;
260
261 /* PCI subsystem metadata. */
262 struct pci_dev *pci_dev;
263
264 /* This device's index into internal_desc->devs. */
265 int dev_idx;
266
267 /* The name of this device, as reported by the kernel. */
268 char kobj_name[GASKET_NAME_MAX];
269
270 /* Virtual address of mapped BAR memory range. */
271 struct gasket_bar_data bar_data[GASKET_NUM_BARS];
272
273 /* Coherent buffer. */
274 struct gasket_coherent_buffer coherent_buffer;
275
276 /* Number of page tables for this device. */
277 int num_page_tables;
278
279 /* Address translations. Page tables have a private implementation. */
280 struct gasket_page_table *page_table[GASKET_MAX_NUM_PAGE_TABLES];
281
282 /* Interrupt data for this device. */
283 struct gasket_interrupt_data *interrupt_data;
284
285 /* Status for this device - GASKET_STATUS_ALIVE or _DEAD. */
286 uint status;
287
288 /* Number of times this device has been reset. */
289 uint reset_count;
290
291 /* Dev information for the cdev node. */
292 struct gasket_cdev_info dev_info;
293
294 /* Hardware revision value for this device. */
295 int hardware_revision;
296
297 /* Protects access to per-device data (i.e. this structure). */
298 struct mutex mutex;
299
300 /* cdev hash tracking/membership structure, Accel and legacy. */
301 /* Unused until Accel is upstreamed. */
302 struct hlist_node hlist_node;
303 struct hlist_node legacy_hlist_node;
304 };
305
306 /* Type of the ioctl handler callback. */
307 typedef long (*gasket_ioctl_handler_cb_t)(struct file *file, uint cmd,
308 void __user *argp);
309 /* Type of the ioctl permissions check callback. See below. */
310 typedef int (*gasket_ioctl_permissions_cb_t)(struct file *filp, uint cmd,
311 void __user *argp);
312
313 /*
314 * Device type descriptor.
315 *
316 * This structure contains device-specific data needed to identify and address a
317 * type of device to be administered via the Gasket generic driver.
318 *
319 * Device IDs are per-driver. In other words, two drivers using the Gasket
320 * framework will each have a distinct device 0 (for example).
321 */
322 struct gasket_driver_desc {
323 /* The name of this device type. */
324 const char *name;
325
326 /* The name of this specific device model. */
327 const char *chip_model;
328
329 /* The version of the chip specified in chip_model. */
330 const char *chip_version;
331
332 /* The version of this driver: "1.0.0", "2.1.3", etc. */
333 const char *driver_version;
334
335 /*
336 * Non-zero if we should create "legacy" (device and device-class-
337 * specific) character devices and sysfs nodes.
338 */
339 /* Unused until Accel is upstreamed. */
340 int legacy_support;
341
342 /* Major and minor numbers identifying the device. */
343 int major, minor;
344
345 /* Module structure for this driver. */
346 struct module *module;
347
348 /* PCI ID table. */
349 const struct pci_device_id *pci_id_table;
350
351 /* The number of page tables handled by this driver. */
352 int num_page_tables;
353
354 /* The index of the bar containing the page tables. */
355 int page_table_bar_index;
356
357 /* Registers used to control each page table. */
358 const struct gasket_page_table_config *page_table_configs;
359
360 /* The bit index indicating whether a PT entry is extended. */
361 int page_table_extended_bit;
362
363 /*
364 * Legacy mmap address adjusment for legacy devices only. Should be 0
365 * for any new device.
366 */
367 ulong legacy_mmap_address_offset;
368
369 /* Set of 6 bar descriptions that describe all PCIe bars.
370 * Note that BUS/AXI devices (i.e. non PCI devices) use those.
371 */
372 struct gasket_bar_desc bar_descriptions[GASKET_NUM_BARS];
373
374 /*
375 * Coherent buffer description.
376 */
377 struct gasket_coherent_buffer_desc coherent_buffer_description;
378
379 /* Interrupt type. (One of gasket_interrupt_type). */
380 int interrupt_type;
381
382 /* Index of the bar containing the interrupt registers to program. */
383 int interrupt_bar_index;
384
385 /* Number of interrupts in the gasket_interrupt_desc array */
386 int num_interrupts;
387
388 /* Description of the interrupts for this device. */
389 const struct gasket_interrupt_desc *interrupts;
390
391 /*
392 * If this device packs multiple interrupt->MSI-X mappings into a
393 * single register (i.e., "uses packed interrupts"), only a single bit
394 * width is supported for each interrupt mapping (unpacked/"full-width"
395 * interrupts are always supported). This value specifies that width. If
396 * packed interrupts are not used, this value is ignored.
397 */
398 int interrupt_pack_width;
399
400 /* Driver callback functions - all may be NULL */
401 /*
402 * device_open_cb: Callback for when a device node is opened in write
403 * mode.
404 * @dev: The gasket_dev struct for this driver instance.
405 *
406 * This callback should perform device-specific setup that needs to
407 * occur only once when a device is first opened.
408 */
409 int (*device_open_cb)(struct gasket_dev *dev);
410
411 /*
412 * device_release_cb: Callback when a device is closed.
413 * @gasket_dev: The gasket_dev struct for this driver instance.
414 *
415 * This callback is called whenever a device node fd is closed, as
416 * opposed to device_close_cb, which is called when the _last_
417 * descriptor for an open file is closed. This call is intended to
418 * handle any per-user or per-fd cleanup.
419 */
420 int (*device_release_cb)(struct gasket_dev *gasket_dev,
421 struct file *file);
422
423 /*
424 * device_close_cb: Callback for when a device node is closed for the
425 * last time.
426 * @dev: The gasket_dev struct for this driver instance.
427 *
428 * This callback should perform device-specific cleanup that only
429 * needs to occur when the last reference to a device node is closed.
430 *
431 * This call is intended to handle and device-wide cleanup, as opposed
432 * to per-fd cleanup (which should be handled by device_release_cb).
433 */
434 int (*device_close_cb)(struct gasket_dev *dev);
435
436 /*
437 * get_mappable_regions_cb: Get descriptors of mappable device memory.
438 * @gasket_dev: Pointer to the struct gasket_dev for this device.
439 * @bar_index: BAR for which to retrieve memory ranges.
440 * @mappable_regions: Out-pointer to the list of mappable regions on the
441 * device/BAR for this process.
442 * @num_mappable_regions: Out-pointer for the size of mappable_regions.
443 *
444 * Called when handling mmap(), this callback is used to determine which
445 * regions of device memory may be mapped by the current process. This
446 * information is then compared to mmap request to determine which
447 * regions to actually map.
448 */
449 int (*get_mappable_regions_cb)(struct gasket_dev *gasket_dev,
450 int bar_index,
451 struct gasket_mappable_region **mappable_regions,
452 int *num_mappable_regions);
453
454 /*
455 * ioctl_permissions_cb: Check permissions for generic ioctls.
456 * @filp: File structure pointer describing this node usage session.
457 * @cmd: ioctl number to handle.
458 * @arg: ioctl-specific data pointer.
459 *
460 * Returns 1 if the ioctl may be executed, 0 otherwise. If this callback
461 * isn't specified a default routine will be used, that only allows the
462 * original device opener (i.e, the "owner") to execute state-affecting
463 * ioctls.
464 */
465 gasket_ioctl_permissions_cb_t ioctl_permissions_cb;
466
467 /*
468 * ioctl_handler_cb: Callback to handle device-specific ioctls.
469 * @filp: File structure pointer describing this node usage session.
470 * @cmd: ioctl number to handle.
471 * @arg: ioctl-specific data pointer.
472 *
473 * Invoked whenever an ioctl is called that the generic Gasket
474 * framework doesn't support. If no cb is registered, unknown ioctls
475 * return -EINVAL. Should return an error status (either -EINVAL or
476 * the error result of the ioctl being handled).
477 */
478 gasket_ioctl_handler_cb_t ioctl_handler_cb;
479
480 /*
481 * device_status_cb: Callback to determine device health.
482 * @dev: Pointer to the gasket_dev struct for this device.
483 *
484 * Called to determine if the device is healthy or not. Should return
485 * a member of the gasket_status_type enum.
486 *
487 */
488 int (*device_status_cb)(struct gasket_dev *dev);
489
490 /*
491 * hardware_revision_cb: Get the device's hardware revision.
492 * @dev: Pointer to the gasket_dev struct for this device.
493 *
494 * Called to determine the reported rev of the physical hardware.
495 * Revision should be >0. A negative return value is an error.
496 */
497 int (*hardware_revision_cb)(struct gasket_dev *dev);
498
499 /*
500 * device_reset_cb: Reset the hardware in question.
501 * @dev: Pointer to the gasket_dev structure for this device.
502 *
503 * Called by reset ioctls. This function should not
504 * lock the gasket_dev mutex. It should return 0 on success
505 * and an error on failure.
506 */
507 int (*device_reset_cb)(struct gasket_dev *dev);
508 };
509
510 /*
511 * Register the specified device type with the framework.
512 * @desc: Populated/initialized device type descriptor.
513 *
514 * This function does _not_ take ownership of desc; the underlying struct must
515 * exist until the matching call to gasket_unregister_device.
516 * This function should be called from your driver's module_init function.
517 */
518 int gasket_register_device(const struct gasket_driver_desc *desc);
519
520 /*
521 * Remove the specified device type from the framework.
522 * @desc: Descriptor for the device type to unregister; it should have been
523 * passed to gasket_register_device in a previous call.
524 *
525 * This function should be called from your driver's module_exit function.
526 */
527 void gasket_unregister_device(const struct gasket_driver_desc *desc);
528
529 /* Add a PCI gasket device. */
530 int gasket_pci_add_device(struct pci_dev *pci_dev,
531 struct gasket_dev **gasket_devp);
532 /* Remove a PCI gasket device. */
533 void gasket_pci_remove_device(struct pci_dev *pci_dev);
534
535 /* Enable a Gasket device. */
536 int gasket_enable_device(struct gasket_dev *gasket_dev);
537
538 /* Disable a Gasket device. */
539 void gasket_disable_device(struct gasket_dev *gasket_dev);
540
541 /*
542 * Reset the Gasket device.
543 * @gasket_dev: Gasket device struct.
544 *
545 * Calls device_reset_cb. Returns 0 on success and an error code othewrise.
546 * gasket_reset_nolock will not lock the mutex, gasket_reset will.
547 *
548 */
549 int gasket_reset(struct gasket_dev *gasket_dev);
550 int gasket_reset_nolock(struct gasket_dev *gasket_dev);
551
552 /*
553 * Memory management functions. These will likely be spun off into their own
554 * file in the future.
555 */
556
557 /* Unmaps the specified mappable region from a VMA. */
558 int gasket_mm_unmap_region(const struct gasket_dev *gasket_dev,
559 struct vm_area_struct *vma,
560 const struct gasket_mappable_region *map_region);
561
562 /*
563 * Get the ioctl permissions callback.
564 * @gasket_dev: Gasket device structure.
565 */
566 gasket_ioctl_permissions_cb_t
567 gasket_get_ioctl_permissions_cb(struct gasket_dev *gasket_dev);
568
569 /**
570 * Lookup a name by number in a num_name table.
571 * @num: Number to lookup.
572 * @table: Array of num_name structures, the table for the lookup.
573 *
574 */
575 const char *gasket_num_name_lookup(uint num,
576 const struct gasket_num_name *table);
577
578 /* Handy inlines */
579 static inline ulong gasket_dev_read_64(struct gasket_dev *gasket_dev, int bar,
580 ulong location)
581 {
582 return readq_relaxed(&gasket_dev->bar_data[bar].virt_base[location]);
583 }
584
585 static inline void gasket_dev_write_64(struct gasket_dev *dev, u64 value,
586 int bar, ulong location)
587 {
588 writeq_relaxed(value, &dev->bar_data[bar].virt_base[location]);
589 }
590
591 static inline void gasket_dev_write_32(struct gasket_dev *dev, u32 value,
592 int bar, ulong location)
593 {
594 writel_relaxed(value, &dev->bar_data[bar].virt_base[location]);
595 }
596
597 static inline u32 gasket_dev_read_32(struct gasket_dev *dev, int bar,
598 ulong location)
599 {
600 return readl_relaxed(&dev->bar_data[bar].virt_base[location]);
601 }
602
603 static inline void gasket_read_modify_write_64(struct gasket_dev *dev, int bar,
604 ulong location, u64 value,
605 u64 mask_width, u64 mask_shift)
606 {
607 u64 mask, tmp;
608
609 tmp = gasket_dev_read_64(dev, bar, location);
610 mask = ((1ULL << mask_width) - 1) << mask_shift;
611 tmp = (tmp & ~mask) | (value << mask_shift);
612 gasket_dev_write_64(dev, tmp, bar, location);
613 }
614
615 static inline void gasket_read_modify_write_32(struct gasket_dev *dev, int bar,
616 ulong location, u32 value,
617 u32 mask_width, u32 mask_shift)
618 {
619 u32 mask, tmp;
620
621 tmp = gasket_dev_read_32(dev, bar, location);
622 mask = ((1 << mask_width) - 1) << mask_shift;
623 tmp = (tmp & ~mask) | (value << mask_shift);
624 gasket_dev_write_32(dev, tmp, bar, location);
625 }
626
627 /* Get the Gasket driver structure for a given device. */
628 const struct gasket_driver_desc *gasket_get_driver_desc(struct gasket_dev *dev);
629
630 /* Get the device structure for a given device. */
631 struct device *gasket_get_device(struct gasket_dev *dev);
632
633 /* Helper function, Asynchronous waits on a given set of bits. */
634 int gasket_wait_with_reschedule(struct gasket_dev *gasket_dev, int bar,
635 u64 offset, u64 mask, u64 val,
636 uint max_retries, u64 delay_ms);
637
638 #endif /* __GASKET_CORE_H__ */