This source file includes following definitions.
- __msu_buffer_entry_find
- msu_buffer_get
- msu_buffer_put
- intel_th_msu_buffer_register
- intel_th_msu_buffer_unregister
- msc_block_is_empty
- msc_win_base_sg
- msc_win_base
- msc_win_base_dma
- msc_win_base_pfn
- msc_is_last_win
- msc_next_window
- msc_win_total_sz
- msc_find_window
- msc_oldest_window
- msc_win_oldest_sg
- msc_iter_bdesc
- msc_iter_install
- msc_iter_remove
- msc_iter_block_start
- msc_iter_win_start
- msc_iter_win_advance
- msc_iter_block_advance
- msc_buffer_iterate
- msc_buffer_clear_hw_header
- intel_th_msu_init
- intel_th_msu_deinit
- msc_win_set_lockout
- msc_configure
- msc_disable
- intel_th_msc_activate
- intel_th_msc_deactivate
- msc_buffer_contig_alloc
- msc_buffer_contig_free
- msc_buffer_contig_get_page
- __msc_buffer_win_alloc
- msc_buffer_set_uc
- msc_buffer_set_wb
- msc_buffer_set_uc
- msc_buffer_set_wb
- msc_buffer_win_alloc
- __msc_buffer_win_free
- msc_buffer_win_free
- msc_buffer_relink
- msc_buffer_multi_free
- msc_buffer_multi_alloc
- msc_buffer_free
- msc_buffer_alloc
- msc_buffer_unlocked_free_unless_used
- msc_buffer_free_unless_used
- msc_buffer_get_page
- msc_win_to_user
- intel_th_msc_open
- intel_th_msc_release
- msc_single_to_user
- intel_th_msc_read
- msc_mmap_open
- msc_mmap_close
- msc_mmap_fault
- intel_th_msc_mmap
- intel_th_msc_wait_empty
- intel_th_msc_init
- msc_win_switch
- intel_th_msc_window_unlock
- msc_work
- intel_th_msc_interrupt
- wrap_show
- wrap_store
- msc_buffer_unassign
- mode_show
- mode_store
- nr_pages_show
- nr_pages_store
- win_switch_store
- intel_th_msc_probe
- intel_th_msc_remove
1
2
3
4
5
6
7
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10 #include <linux/types.h>
11 #include <linux/module.h>
12 #include <linux/device.h>
13 #include <linux/uaccess.h>
14 #include <linux/sizes.h>
15 #include <linux/printk.h>
16 #include <linux/slab.h>
17 #include <linux/mm.h>
18 #include <linux/fs.h>
19 #include <linux/io.h>
20 #include <linux/workqueue.h>
21 #include <linux/dma-mapping.h>
22
23 #ifdef CONFIG_X86
24 #include <asm/set_memory.h>
25 #endif
26
27 #include <linux/intel_th.h>
28 #include "intel_th.h"
29 #include "msu.h"
30
31 #define msc_dev(x) (&(x)->thdev->dev)
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50 enum lockout_state {
51 WIN_READY = 0,
52 WIN_INUSE,
53 WIN_LOCKED
54 };
55
56
57
58
59
60
61
62
63
64
65
66
67 struct msc_window {
68 struct list_head entry;
69 unsigned long pgoff;
70 enum lockout_state lockout;
71 spinlock_t lo_lock;
72 unsigned int nr_blocks;
73 unsigned int nr_segs;
74 struct msc *msc;
75 struct sg_table _sgt;
76 struct sg_table *sgt;
77 };
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92 struct msc_iter {
93 struct list_head entry;
94 struct msc *msc;
95 struct msc_window *start_win;
96 struct msc_window *win;
97 unsigned long offset;
98 struct scatterlist *start_block;
99 struct scatterlist *block;
100 unsigned int block_off;
101 unsigned int wrap_count;
102 unsigned int eof;
103 };
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129 struct msc {
130 void __iomem *reg_base;
131 void __iomem *msu_base;
132 struct intel_th_device *thdev;
133
134 const struct msu_buffer *mbuf;
135 void *mbuf_priv;
136
137 struct work_struct work;
138 struct list_head win_list;
139 struct sg_table single_sgt;
140 struct msc_window *cur_win;
141 unsigned long nr_pages;
142 unsigned long single_sz;
143 unsigned int single_wrap : 1;
144 void *base;
145 dma_addr_t base_addr;
146 u32 orig_addr;
147 u32 orig_sz;
148
149
150 atomic_t user_count;
151
152 atomic_t mmap_count;
153 struct mutex buf_mutex;
154
155 struct list_head iter_list;
156
157
158 unsigned int enabled : 1,
159 wrap : 1,
160 do_irq : 1;
161 unsigned int mode;
162 unsigned int burst_len;
163 unsigned int index;
164 };
165
166 static LIST_HEAD(msu_buffer_list);
167 static DEFINE_MUTEX(msu_buffer_mutex);
168
169
170
171
172
173
174
175 struct msu_buffer_entry {
176 struct list_head entry;
177 const struct msu_buffer *mbuf;
178 struct module *owner;
179 };
180
181 static struct msu_buffer_entry *__msu_buffer_entry_find(const char *name)
182 {
183 struct msu_buffer_entry *mbe;
184
185 lockdep_assert_held(&msu_buffer_mutex);
186
187 list_for_each_entry(mbe, &msu_buffer_list, entry) {
188 if (!strcmp(mbe->mbuf->name, name))
189 return mbe;
190 }
191
192 return NULL;
193 }
194
195 static const struct msu_buffer *
196 msu_buffer_get(const char *name)
197 {
198 struct msu_buffer_entry *mbe;
199
200 mutex_lock(&msu_buffer_mutex);
201 mbe = __msu_buffer_entry_find(name);
202 if (mbe && !try_module_get(mbe->owner))
203 mbe = NULL;
204 mutex_unlock(&msu_buffer_mutex);
205
206 return mbe ? mbe->mbuf : NULL;
207 }
208
209 static void msu_buffer_put(const struct msu_buffer *mbuf)
210 {
211 struct msu_buffer_entry *mbe;
212
213 mutex_lock(&msu_buffer_mutex);
214 mbe = __msu_buffer_entry_find(mbuf->name);
215 if (mbe)
216 module_put(mbe->owner);
217 mutex_unlock(&msu_buffer_mutex);
218 }
219
220 int intel_th_msu_buffer_register(const struct msu_buffer *mbuf,
221 struct module *owner)
222 {
223 struct msu_buffer_entry *mbe;
224 int ret = 0;
225
226 mbe = kzalloc(sizeof(*mbe), GFP_KERNEL);
227 if (!mbe)
228 return -ENOMEM;
229
230 mutex_lock(&msu_buffer_mutex);
231 if (__msu_buffer_entry_find(mbuf->name)) {
232 ret = -EEXIST;
233 kfree(mbe);
234 goto unlock;
235 }
236
237 mbe->mbuf = mbuf;
238 mbe->owner = owner;
239 list_add_tail(&mbe->entry, &msu_buffer_list);
240 unlock:
241 mutex_unlock(&msu_buffer_mutex);
242
243 return ret;
244 }
245 EXPORT_SYMBOL_GPL(intel_th_msu_buffer_register);
246
247 void intel_th_msu_buffer_unregister(const struct msu_buffer *mbuf)
248 {
249 struct msu_buffer_entry *mbe;
250
251 mutex_lock(&msu_buffer_mutex);
252 mbe = __msu_buffer_entry_find(mbuf->name);
253 if (mbe) {
254 list_del(&mbe->entry);
255 kfree(mbe);
256 }
257 mutex_unlock(&msu_buffer_mutex);
258 }
259 EXPORT_SYMBOL_GPL(intel_th_msu_buffer_unregister);
260
261 static inline bool msc_block_is_empty(struct msc_block_desc *bdesc)
262 {
263
264 if (!bdesc->valid_dw)
265 return true;
266
267
268 if (!msc_data_sz(bdesc))
269 return true;
270
271 return false;
272 }
273
274 static inline struct scatterlist *msc_win_base_sg(struct msc_window *win)
275 {
276 return win->sgt->sgl;
277 }
278
279 static inline struct msc_block_desc *msc_win_base(struct msc_window *win)
280 {
281 return sg_virt(msc_win_base_sg(win));
282 }
283
284 static inline dma_addr_t msc_win_base_dma(struct msc_window *win)
285 {
286 return sg_dma_address(msc_win_base_sg(win));
287 }
288
289 static inline unsigned long
290 msc_win_base_pfn(struct msc_window *win)
291 {
292 return PFN_DOWN(msc_win_base_dma(win));
293 }
294
295
296
297
298
299
300 static inline bool msc_is_last_win(struct msc_window *win)
301 {
302 return win->entry.next == &win->msc->win_list;
303 }
304
305
306
307
308
309
310
311 static struct msc_window *msc_next_window(struct msc_window *win)
312 {
313 if (msc_is_last_win(win))
314 return list_first_entry(&win->msc->win_list, struct msc_window,
315 entry);
316
317 return list_next_entry(win, entry);
318 }
319
320 static size_t msc_win_total_sz(struct msc_window *win)
321 {
322 struct scatterlist *sg;
323 unsigned int blk;
324 size_t size = 0;
325
326 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
327 struct msc_block_desc *bdesc = sg_virt(sg);
328
329 if (msc_block_wrapped(bdesc))
330 return (size_t)win->nr_blocks << PAGE_SHIFT;
331
332 size += msc_total_sz(bdesc);
333 if (msc_block_last_written(bdesc))
334 break;
335 }
336
337 return size;
338 }
339
340
341
342
343
344
345
346
347
348
349 static struct msc_window *
350 msc_find_window(struct msc *msc, struct sg_table *sgt, bool nonempty)
351 {
352 struct msc_window *win;
353 unsigned int found = 0;
354
355 if (list_empty(&msc->win_list))
356 return NULL;
357
358
359
360
361
362
363 list_for_each_entry(win, &msc->win_list, entry) {
364 if (win->sgt == sgt)
365 found++;
366
367
368 if (nonempty && msc_block_is_empty(msc_win_base(win)))
369 continue;
370
371 if (found)
372 return win;
373 }
374
375 return NULL;
376 }
377
378
379
380
381
382
383
384
385
386
387 static struct msc_window *msc_oldest_window(struct msc *msc)
388 {
389 struct msc_window *win;
390
391 if (list_empty(&msc->win_list))
392 return NULL;
393
394 win = msc_find_window(msc, msc_next_window(msc->cur_win)->sgt, true);
395 if (win)
396 return win;
397
398 return list_first_entry(&msc->win_list, struct msc_window, entry);
399 }
400
401
402
403
404
405
406
407 static struct scatterlist *msc_win_oldest_sg(struct msc_window *win)
408 {
409 unsigned int blk;
410 struct scatterlist *sg;
411 struct msc_block_desc *bdesc = msc_win_base(win);
412
413
414 if (!msc_block_wrapped(bdesc))
415 return msc_win_base_sg(win);
416
417
418
419
420
421 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
422 struct msc_block_desc *bdesc = sg_virt(sg);
423
424 if (msc_block_last_written(bdesc))
425 return sg;
426 }
427
428 return msc_win_base_sg(win);
429 }
430
431 static struct msc_block_desc *msc_iter_bdesc(struct msc_iter *iter)
432 {
433 return sg_virt(iter->block);
434 }
435
436 static struct msc_iter *msc_iter_install(struct msc *msc)
437 {
438 struct msc_iter *iter;
439
440 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
441 if (!iter)
442 return ERR_PTR(-ENOMEM);
443
444 mutex_lock(&msc->buf_mutex);
445
446
447
448
449
450
451
452 if (msc->enabled) {
453 kfree(iter);
454 iter = ERR_PTR(-EBUSY);
455 goto unlock;
456 }
457
458 iter->msc = msc;
459
460 list_add_tail(&iter->entry, &msc->iter_list);
461 unlock:
462 mutex_unlock(&msc->buf_mutex);
463
464 return iter;
465 }
466
467 static void msc_iter_remove(struct msc_iter *iter, struct msc *msc)
468 {
469 mutex_lock(&msc->buf_mutex);
470 list_del(&iter->entry);
471 mutex_unlock(&msc->buf_mutex);
472
473 kfree(iter);
474 }
475
476 static void msc_iter_block_start(struct msc_iter *iter)
477 {
478 if (iter->start_block)
479 return;
480
481 iter->start_block = msc_win_oldest_sg(iter->win);
482 iter->block = iter->start_block;
483 iter->wrap_count = 0;
484
485
486
487
488
489 if (msc_block_wrapped(msc_iter_bdesc(iter)))
490 iter->wrap_count = 2;
491
492 }
493
494 static int msc_iter_win_start(struct msc_iter *iter, struct msc *msc)
495 {
496
497 if (iter->start_win)
498 return 0;
499
500 iter->start_win = msc_oldest_window(msc);
501 if (!iter->start_win)
502 return -EINVAL;
503
504 iter->win = iter->start_win;
505 iter->start_block = NULL;
506
507 msc_iter_block_start(iter);
508
509 return 0;
510 }
511
512 static int msc_iter_win_advance(struct msc_iter *iter)
513 {
514 iter->win = msc_next_window(iter->win);
515 iter->start_block = NULL;
516
517 if (iter->win == iter->start_win) {
518 iter->eof++;
519 return 1;
520 }
521
522 msc_iter_block_start(iter);
523
524 return 0;
525 }
526
527 static int msc_iter_block_advance(struct msc_iter *iter)
528 {
529 iter->block_off = 0;
530
531
532 if (iter->wrap_count && iter->block == iter->start_block) {
533 iter->wrap_count--;
534 if (!iter->wrap_count)
535
536 return msc_iter_win_advance(iter);
537 }
538
539
540 if (!iter->wrap_count && msc_block_last_written(msc_iter_bdesc(iter)))
541
542 return msc_iter_win_advance(iter);
543
544
545 if (sg_is_last(iter->block))
546 iter->block = msc_win_base_sg(iter->win);
547 else
548 iter->block = sg_next(iter->block);
549
550
551 if (!iter->wrap_count && iter->block == iter->start_block)
552 return msc_iter_win_advance(iter);
553
554 return 0;
555 }
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573 static ssize_t
574 msc_buffer_iterate(struct msc_iter *iter, size_t size, void *data,
575 unsigned long (*fn)(void *, void *, size_t))
576 {
577 struct msc *msc = iter->msc;
578 size_t len = size;
579 unsigned int advance;
580
581 if (iter->eof)
582 return 0;
583
584
585 if (msc_iter_win_start(iter, msc))
586 return 0;
587
588 do {
589 unsigned long data_bytes = msc_data_sz(msc_iter_bdesc(iter));
590 void *src = (void *)msc_iter_bdesc(iter) + MSC_BDESC;
591 size_t tocopy = data_bytes, copied = 0;
592 size_t remaining = 0;
593
594 advance = 1;
595
596
597
598
599
600
601
602
603
604
605
606
607
608 if (iter->block == iter->start_block && iter->wrap_count == 2) {
609 tocopy = DATA_IN_PAGE - data_bytes;
610 src += data_bytes;
611 }
612
613 if (!tocopy)
614 goto next_block;
615
616 tocopy -= iter->block_off;
617 src += iter->block_off;
618
619 if (len < tocopy) {
620 tocopy = len;
621 advance = 0;
622 }
623
624 remaining = fn(data, src, tocopy);
625
626 if (remaining)
627 advance = 0;
628
629 copied = tocopy - remaining;
630 len -= copied;
631 iter->block_off += copied;
632 iter->offset += copied;
633
634 if (!advance)
635 break;
636
637 next_block:
638 if (msc_iter_block_advance(iter))
639 break;
640
641 } while (len);
642
643 return size - len;
644 }
645
646
647
648
649
650 static void msc_buffer_clear_hw_header(struct msc *msc)
651 {
652 struct msc_window *win;
653 struct scatterlist *sg;
654
655 list_for_each_entry(win, &msc->win_list, entry) {
656 unsigned int blk;
657 size_t hw_sz = sizeof(struct msc_block_desc) -
658 offsetof(struct msc_block_desc, hw_tag);
659
660 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
661 struct msc_block_desc *bdesc = sg_virt(sg);
662
663 memset(&bdesc->hw_tag, 0, hw_sz);
664 }
665 }
666 }
667
668 static int intel_th_msu_init(struct msc *msc)
669 {
670 u32 mintctl, msusts;
671
672 if (!msc->do_irq)
673 return 0;
674
675 if (!msc->mbuf)
676 return 0;
677
678 mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL);
679 mintctl |= msc->index ? M1BLIE : M0BLIE;
680 iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL);
681 if (mintctl != ioread32(msc->msu_base + REG_MSU_MINTCTL)) {
682 dev_info(msc_dev(msc), "MINTCTL ignores writes: no usable interrupts\n");
683 msc->do_irq = 0;
684 return 0;
685 }
686
687 msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS);
688 iowrite32(msusts, msc->msu_base + REG_MSU_MSUSTS);
689
690 return 0;
691 }
692
693 static void intel_th_msu_deinit(struct msc *msc)
694 {
695 u32 mintctl;
696
697 if (!msc->do_irq)
698 return;
699
700 mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL);
701 mintctl &= msc->index ? ~M1BLIE : ~M0BLIE;
702 iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL);
703 }
704
705 static int msc_win_set_lockout(struct msc_window *win,
706 enum lockout_state expect,
707 enum lockout_state new)
708 {
709 enum lockout_state old;
710 unsigned long flags;
711 int ret = 0;
712
713 if (!win->msc->mbuf)
714 return 0;
715
716 spin_lock_irqsave(&win->lo_lock, flags);
717 old = win->lockout;
718
719 if (old != expect) {
720 ret = -EINVAL;
721 goto unlock;
722 }
723
724 win->lockout = new;
725
726 if (old == expect && new == WIN_LOCKED)
727 atomic_inc(&win->msc->user_count);
728 else if (old == expect && old == WIN_LOCKED)
729 atomic_dec(&win->msc->user_count);
730
731 unlock:
732 spin_unlock_irqrestore(&win->lo_lock, flags);
733
734 if (ret) {
735 if (expect == WIN_READY && old == WIN_LOCKED)
736 return -EBUSY;
737
738
739 if (expect == WIN_LOCKED && old == new)
740 return 0;
741
742 dev_warn_ratelimited(msc_dev(win->msc),
743 "expected lockout state %d, got %d\n",
744 expect, old);
745 }
746
747 return ret;
748 }
749
750
751
752
753
754
755
756
757 static int msc_configure(struct msc *msc)
758 {
759 u32 reg;
760
761 lockdep_assert_held(&msc->buf_mutex);
762
763 if (msc->mode > MSC_MODE_MULTI)
764 return -EINVAL;
765
766 if (msc->mode == MSC_MODE_MULTI) {
767 if (msc_win_set_lockout(msc->cur_win, WIN_READY, WIN_INUSE))
768 return -EBUSY;
769
770 msc_buffer_clear_hw_header(msc);
771 }
772
773 msc->orig_addr = ioread32(msc->reg_base + REG_MSU_MSC0BAR);
774 msc->orig_sz = ioread32(msc->reg_base + REG_MSU_MSC0SIZE);
775
776 reg = msc->base_addr >> PAGE_SHIFT;
777 iowrite32(reg, msc->reg_base + REG_MSU_MSC0BAR);
778
779 if (msc->mode == MSC_MODE_SINGLE) {
780 reg = msc->nr_pages;
781 iowrite32(reg, msc->reg_base + REG_MSU_MSC0SIZE);
782 }
783
784 reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL);
785 reg &= ~(MSC_MODE | MSC_WRAPEN | MSC_EN | MSC_RD_HDR_OVRD);
786
787 reg |= MSC_EN;
788 reg |= msc->mode << __ffs(MSC_MODE);
789 reg |= msc->burst_len << __ffs(MSC_LEN);
790
791 if (msc->wrap)
792 reg |= MSC_WRAPEN;
793
794 iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL);
795
796 intel_th_msu_init(msc);
797
798 msc->thdev->output.multiblock = msc->mode == MSC_MODE_MULTI;
799 intel_th_trace_enable(msc->thdev);
800 msc->enabled = 1;
801
802 if (msc->mbuf && msc->mbuf->activate)
803 msc->mbuf->activate(msc->mbuf_priv);
804
805 return 0;
806 }
807
808
809
810
811
812
813
814
815 static void msc_disable(struct msc *msc)
816 {
817 struct msc_window *win = msc->cur_win;
818 u32 reg;
819
820 lockdep_assert_held(&msc->buf_mutex);
821
822 if (msc->mode == MSC_MODE_MULTI)
823 msc_win_set_lockout(win, WIN_INUSE, WIN_LOCKED);
824
825 if (msc->mbuf && msc->mbuf->deactivate)
826 msc->mbuf->deactivate(msc->mbuf_priv);
827 intel_th_msu_deinit(msc);
828 intel_th_trace_disable(msc->thdev);
829
830 if (msc->mode == MSC_MODE_SINGLE) {
831 reg = ioread32(msc->reg_base + REG_MSU_MSC0STS);
832 msc->single_wrap = !!(reg & MSCSTS_WRAPSTAT);
833
834 reg = ioread32(msc->reg_base + REG_MSU_MSC0MWP);
835 msc->single_sz = reg & ((msc->nr_pages << PAGE_SHIFT) - 1);
836 dev_dbg(msc_dev(msc), "MSCnMWP: %08x/%08lx, wrap: %d\n",
837 reg, msc->single_sz, msc->single_wrap);
838 }
839
840 reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL);
841 reg &= ~MSC_EN;
842 iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL);
843
844 if (msc->mbuf && msc->mbuf->ready)
845 msc->mbuf->ready(msc->mbuf_priv, win->sgt,
846 msc_win_total_sz(win));
847
848 msc->enabled = 0;
849
850 iowrite32(msc->orig_addr, msc->reg_base + REG_MSU_MSC0BAR);
851 iowrite32(msc->orig_sz, msc->reg_base + REG_MSU_MSC0SIZE);
852
853 dev_dbg(msc_dev(msc), "MSCnNWSA: %08x\n",
854 ioread32(msc->reg_base + REG_MSU_MSC0NWSA));
855
856 reg = ioread32(msc->reg_base + REG_MSU_MSC0STS);
857 dev_dbg(msc_dev(msc), "MSCnSTS: %08x\n", reg);
858
859 reg = ioread32(msc->reg_base + REG_MSU_MSUSTS);
860 reg &= msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST;
861 iowrite32(reg, msc->reg_base + REG_MSU_MSUSTS);
862 }
863
864 static int intel_th_msc_activate(struct intel_th_device *thdev)
865 {
866 struct msc *msc = dev_get_drvdata(&thdev->dev);
867 int ret = -EBUSY;
868
869 if (!atomic_inc_unless_negative(&msc->user_count))
870 return -ENODEV;
871
872 mutex_lock(&msc->buf_mutex);
873
874
875 if (list_empty(&msc->iter_list))
876 ret = msc_configure(msc);
877
878 mutex_unlock(&msc->buf_mutex);
879
880 if (ret)
881 atomic_dec(&msc->user_count);
882
883 return ret;
884 }
885
886 static void intel_th_msc_deactivate(struct intel_th_device *thdev)
887 {
888 struct msc *msc = dev_get_drvdata(&thdev->dev);
889
890 mutex_lock(&msc->buf_mutex);
891 if (msc->enabled) {
892 msc_disable(msc);
893 atomic_dec(&msc->user_count);
894 }
895 mutex_unlock(&msc->buf_mutex);
896 }
897
898
899
900
901
902
903
904
905
906
907
908 static int msc_buffer_contig_alloc(struct msc *msc, unsigned long size)
909 {
910 unsigned long nr_pages = size >> PAGE_SHIFT;
911 unsigned int order = get_order(size);
912 struct page *page;
913 int ret;
914
915 if (!size)
916 return 0;
917
918 ret = sg_alloc_table(&msc->single_sgt, 1, GFP_KERNEL);
919 if (ret)
920 goto err_out;
921
922 ret = -ENOMEM;
923 page = alloc_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32, order);
924 if (!page)
925 goto err_free_sgt;
926
927 split_page(page, order);
928 sg_set_buf(msc->single_sgt.sgl, page_address(page), size);
929
930 ret = dma_map_sg(msc_dev(msc)->parent->parent, msc->single_sgt.sgl, 1,
931 DMA_FROM_DEVICE);
932 if (ret < 0)
933 goto err_free_pages;
934
935 msc->nr_pages = nr_pages;
936 msc->base = page_address(page);
937 msc->base_addr = sg_dma_address(msc->single_sgt.sgl);
938
939 return 0;
940
941 err_free_pages:
942 __free_pages(page, order);
943
944 err_free_sgt:
945 sg_free_table(&msc->single_sgt);
946
947 err_out:
948 return ret;
949 }
950
951
952
953
954
955 static void msc_buffer_contig_free(struct msc *msc)
956 {
957 unsigned long off;
958
959 dma_unmap_sg(msc_dev(msc)->parent->parent, msc->single_sgt.sgl,
960 1, DMA_FROM_DEVICE);
961 sg_free_table(&msc->single_sgt);
962
963 for (off = 0; off < msc->nr_pages << PAGE_SHIFT; off += PAGE_SIZE) {
964 struct page *page = virt_to_page(msc->base + off);
965
966 page->mapping = NULL;
967 __free_page(page);
968 }
969
970 msc->nr_pages = 0;
971 }
972
973
974
975
976
977
978
979
980 static struct page *msc_buffer_contig_get_page(struct msc *msc,
981 unsigned long pgoff)
982 {
983 if (pgoff >= msc->nr_pages)
984 return NULL;
985
986 return virt_to_page(msc->base + (pgoff << PAGE_SHIFT));
987 }
988
989 static int __msc_buffer_win_alloc(struct msc_window *win,
990 unsigned int nr_segs)
991 {
992 struct scatterlist *sg_ptr;
993 void *block;
994 int i, ret;
995
996 ret = sg_alloc_table(win->sgt, nr_segs, GFP_KERNEL);
997 if (ret)
998 return -ENOMEM;
999
1000 for_each_sg(win->sgt->sgl, sg_ptr, nr_segs, i) {
1001 block = dma_alloc_coherent(msc_dev(win->msc)->parent->parent,
1002 PAGE_SIZE, &sg_dma_address(sg_ptr),
1003 GFP_KERNEL);
1004 if (!block)
1005 goto err_nomem;
1006
1007 sg_set_buf(sg_ptr, block, PAGE_SIZE);
1008 }
1009
1010 return nr_segs;
1011
1012 err_nomem:
1013 for_each_sg(win->sgt->sgl, sg_ptr, i, ret)
1014 dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE,
1015 sg_virt(sg_ptr), sg_dma_address(sg_ptr));
1016
1017 sg_free_table(win->sgt);
1018
1019 return -ENOMEM;
1020 }
1021
1022 #ifdef CONFIG_X86
1023 static void msc_buffer_set_uc(struct msc_window *win, unsigned int nr_segs)
1024 {
1025 struct scatterlist *sg_ptr;
1026 int i;
1027
1028 for_each_sg(win->sgt->sgl, sg_ptr, nr_segs, i) {
1029
1030 set_memory_uc((unsigned long)sg_virt(sg_ptr),
1031 PFN_DOWN(sg_ptr->length));
1032 }
1033 }
1034
1035 static void msc_buffer_set_wb(struct msc_window *win)
1036 {
1037 struct scatterlist *sg_ptr;
1038 int i;
1039
1040 for_each_sg(win->sgt->sgl, sg_ptr, win->nr_segs, i) {
1041
1042 set_memory_wb((unsigned long)sg_virt(sg_ptr),
1043 PFN_DOWN(sg_ptr->length));
1044 }
1045 }
1046 #else
1047 static inline void
1048 msc_buffer_set_uc(struct msc_window *win, unsigned int nr_segs) {}
1049 static inline void msc_buffer_set_wb(struct msc_window *win) {}
1050 #endif
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062 static int msc_buffer_win_alloc(struct msc *msc, unsigned int nr_blocks)
1063 {
1064 struct msc_window *win;
1065 int ret = -ENOMEM;
1066
1067 if (!nr_blocks)
1068 return 0;
1069
1070 win = kzalloc(sizeof(*win), GFP_KERNEL);
1071 if (!win)
1072 return -ENOMEM;
1073
1074 win->msc = msc;
1075 win->sgt = &win->_sgt;
1076 win->lockout = WIN_READY;
1077 spin_lock_init(&win->lo_lock);
1078
1079 if (!list_empty(&msc->win_list)) {
1080 struct msc_window *prev = list_last_entry(&msc->win_list,
1081 struct msc_window,
1082 entry);
1083
1084 win->pgoff = prev->pgoff + prev->nr_blocks;
1085 }
1086
1087 if (msc->mbuf && msc->mbuf->alloc_window)
1088 ret = msc->mbuf->alloc_window(msc->mbuf_priv, &win->sgt,
1089 nr_blocks << PAGE_SHIFT);
1090 else
1091 ret = __msc_buffer_win_alloc(win, nr_blocks);
1092
1093 if (ret <= 0)
1094 goto err_nomem;
1095
1096 msc_buffer_set_uc(win, ret);
1097
1098 win->nr_segs = ret;
1099 win->nr_blocks = nr_blocks;
1100
1101 if (list_empty(&msc->win_list)) {
1102 msc->base = msc_win_base(win);
1103 msc->base_addr = msc_win_base_dma(win);
1104 msc->cur_win = win;
1105 }
1106
1107 list_add_tail(&win->entry, &msc->win_list);
1108 msc->nr_pages += nr_blocks;
1109
1110 return 0;
1111
1112 err_nomem:
1113 kfree(win);
1114
1115 return ret;
1116 }
1117
1118 static void __msc_buffer_win_free(struct msc *msc, struct msc_window *win)
1119 {
1120 struct scatterlist *sg;
1121 int i;
1122
1123 for_each_sg(win->sgt->sgl, sg, win->nr_segs, i) {
1124 struct page *page = sg_page(sg);
1125
1126 page->mapping = NULL;
1127 dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE,
1128 sg_virt(sg), sg_dma_address(sg));
1129 }
1130 sg_free_table(win->sgt);
1131 }
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141 static void msc_buffer_win_free(struct msc *msc, struct msc_window *win)
1142 {
1143 msc->nr_pages -= win->nr_blocks;
1144
1145 list_del(&win->entry);
1146 if (list_empty(&msc->win_list)) {
1147 msc->base = NULL;
1148 msc->base_addr = 0;
1149 }
1150
1151 msc_buffer_set_wb(win);
1152
1153 if (msc->mbuf && msc->mbuf->free_window)
1154 msc->mbuf->free_window(msc->mbuf_priv, win->sgt);
1155 else
1156 __msc_buffer_win_free(msc, win);
1157
1158 kfree(win);
1159 }
1160
1161
1162
1163
1164
1165
1166
1167
1168 static void msc_buffer_relink(struct msc *msc)
1169 {
1170 struct msc_window *win, *next_win;
1171
1172
1173 list_for_each_entry(win, &msc->win_list, entry) {
1174 struct scatterlist *sg;
1175 unsigned int blk;
1176 u32 sw_tag = 0;
1177
1178
1179
1180
1181
1182 if (msc_is_last_win(win)) {
1183 sw_tag |= MSC_SW_TAG_LASTWIN;
1184 next_win = list_first_entry(&msc->win_list,
1185 struct msc_window, entry);
1186 } else {
1187 next_win = list_next_entry(win, entry);
1188 }
1189
1190 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
1191 struct msc_block_desc *bdesc = sg_virt(sg);
1192
1193 memset(bdesc, 0, sizeof(*bdesc));
1194
1195 bdesc->next_win = msc_win_base_pfn(next_win);
1196
1197
1198
1199
1200
1201 if (blk == win->nr_segs - 1) {
1202 sw_tag |= MSC_SW_TAG_LASTBLK;
1203 bdesc->next_blk = msc_win_base_pfn(win);
1204 } else {
1205 dma_addr_t addr = sg_dma_address(sg_next(sg));
1206
1207 bdesc->next_blk = PFN_DOWN(addr);
1208 }
1209
1210 bdesc->sw_tag = sw_tag;
1211 bdesc->block_sz = sg->length / 64;
1212 }
1213 }
1214
1215
1216
1217
1218
1219 wmb();
1220 }
1221
1222 static void msc_buffer_multi_free(struct msc *msc)
1223 {
1224 struct msc_window *win, *iter;
1225
1226 list_for_each_entry_safe(win, iter, &msc->win_list, entry)
1227 msc_buffer_win_free(msc, win);
1228 }
1229
1230 static int msc_buffer_multi_alloc(struct msc *msc, unsigned long *nr_pages,
1231 unsigned int nr_wins)
1232 {
1233 int ret, i;
1234
1235 for (i = 0; i < nr_wins; i++) {
1236 ret = msc_buffer_win_alloc(msc, nr_pages[i]);
1237 if (ret) {
1238 msc_buffer_multi_free(msc);
1239 return ret;
1240 }
1241 }
1242
1243 msc_buffer_relink(msc);
1244
1245 return 0;
1246 }
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257 static void msc_buffer_free(struct msc *msc)
1258 {
1259 if (msc->mode == MSC_MODE_SINGLE)
1260 msc_buffer_contig_free(msc);
1261 else if (msc->mode == MSC_MODE_MULTI)
1262 msc_buffer_multi_free(msc);
1263 }
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281 static int msc_buffer_alloc(struct msc *msc, unsigned long *nr_pages,
1282 unsigned int nr_wins)
1283 {
1284 int ret;
1285
1286
1287 if (atomic_read(&msc->user_count) != -1)
1288 return -EBUSY;
1289
1290 if (msc->mode == MSC_MODE_SINGLE) {
1291 if (nr_wins != 1)
1292 return -EINVAL;
1293
1294 ret = msc_buffer_contig_alloc(msc, nr_pages[0] << PAGE_SHIFT);
1295 } else if (msc->mode == MSC_MODE_MULTI) {
1296 ret = msc_buffer_multi_alloc(msc, nr_pages, nr_wins);
1297 } else {
1298 ret = -EINVAL;
1299 }
1300
1301 if (!ret) {
1302
1303 smp_mb__before_atomic();
1304
1305 if (WARN_ON_ONCE(atomic_cmpxchg(&msc->user_count, -1, 0) != -1))
1306 return -EINVAL;
1307 }
1308
1309 return ret;
1310 }
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323 static int msc_buffer_unlocked_free_unless_used(struct msc *msc)
1324 {
1325 int count, ret = 0;
1326
1327 count = atomic_cmpxchg(&msc->user_count, 0, -1);
1328
1329
1330 if (count > 0)
1331 ret = -EBUSY;
1332
1333 else if (!count)
1334 msc_buffer_free(msc);
1335
1336
1337 return ret;
1338 }
1339
1340
1341
1342
1343
1344
1345
1346 static int msc_buffer_free_unless_used(struct msc *msc)
1347 {
1348 int ret;
1349
1350 mutex_lock(&msc->buf_mutex);
1351 ret = msc_buffer_unlocked_free_unless_used(msc);
1352 mutex_unlock(&msc->buf_mutex);
1353
1354 return ret;
1355 }
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367 static struct page *msc_buffer_get_page(struct msc *msc, unsigned long pgoff)
1368 {
1369 struct msc_window *win;
1370 struct scatterlist *sg;
1371 unsigned int blk;
1372
1373 if (msc->mode == MSC_MODE_SINGLE)
1374 return msc_buffer_contig_get_page(msc, pgoff);
1375
1376 list_for_each_entry(win, &msc->win_list, entry)
1377 if (pgoff >= win->pgoff && pgoff < win->pgoff + win->nr_blocks)
1378 goto found;
1379
1380 return NULL;
1381
1382 found:
1383 pgoff -= win->pgoff;
1384
1385 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
1386 struct page *page = sg_page(sg);
1387 size_t pgsz = PFN_DOWN(sg->length);
1388
1389 if (pgoff < pgsz)
1390 return page + pgoff;
1391
1392 pgoff -= pgsz;
1393 }
1394
1395 return NULL;
1396 }
1397
1398
1399
1400
1401
1402
1403 struct msc_win_to_user_struct {
1404 char __user *buf;
1405 unsigned long offset;
1406 };
1407
1408
1409
1410
1411
1412
1413
1414 static unsigned long msc_win_to_user(void *data, void *src, size_t len)
1415 {
1416 struct msc_win_to_user_struct *u = data;
1417 unsigned long ret;
1418
1419 ret = copy_to_user(u->buf + u->offset, src, len);
1420 u->offset += len - ret;
1421
1422 return ret;
1423 }
1424
1425
1426
1427
1428
1429
1430 static int intel_th_msc_open(struct inode *inode, struct file *file)
1431 {
1432 struct intel_th_device *thdev = file->private_data;
1433 struct msc *msc = dev_get_drvdata(&thdev->dev);
1434 struct msc_iter *iter;
1435
1436 if (!capable(CAP_SYS_RAWIO))
1437 return -EPERM;
1438
1439 iter = msc_iter_install(msc);
1440 if (IS_ERR(iter))
1441 return PTR_ERR(iter);
1442
1443 file->private_data = iter;
1444
1445 return nonseekable_open(inode, file);
1446 }
1447
1448 static int intel_th_msc_release(struct inode *inode, struct file *file)
1449 {
1450 struct msc_iter *iter = file->private_data;
1451 struct msc *msc = iter->msc;
1452
1453 msc_iter_remove(iter, msc);
1454
1455 return 0;
1456 }
1457
1458 static ssize_t
1459 msc_single_to_user(struct msc *msc, char __user *buf, loff_t off, size_t len)
1460 {
1461 unsigned long size = msc->nr_pages << PAGE_SHIFT, rem = len;
1462 unsigned long start = off, tocopy = 0;
1463
1464 if (msc->single_wrap) {
1465 start += msc->single_sz;
1466 if (start < size) {
1467 tocopy = min(rem, size - start);
1468 if (copy_to_user(buf, msc->base + start, tocopy))
1469 return -EFAULT;
1470
1471 buf += tocopy;
1472 rem -= tocopy;
1473 start += tocopy;
1474 }
1475
1476 start &= size - 1;
1477 if (rem) {
1478 tocopy = min(rem, msc->single_sz - start);
1479 if (copy_to_user(buf, msc->base + start, tocopy))
1480 return -EFAULT;
1481
1482 rem -= tocopy;
1483 }
1484
1485 return len - rem;
1486 }
1487
1488 if (copy_to_user(buf, msc->base + start, rem))
1489 return -EFAULT;
1490
1491 return len;
1492 }
1493
1494 static ssize_t intel_th_msc_read(struct file *file, char __user *buf,
1495 size_t len, loff_t *ppos)
1496 {
1497 struct msc_iter *iter = file->private_data;
1498 struct msc *msc = iter->msc;
1499 size_t size;
1500 loff_t off = *ppos;
1501 ssize_t ret = 0;
1502
1503 if (!atomic_inc_unless_negative(&msc->user_count))
1504 return 0;
1505
1506 if (msc->mode == MSC_MODE_SINGLE && !msc->single_wrap)
1507 size = msc->single_sz;
1508 else
1509 size = msc->nr_pages << PAGE_SHIFT;
1510
1511 if (!size)
1512 goto put_count;
1513
1514 if (off >= size)
1515 goto put_count;
1516
1517 if (off + len >= size)
1518 len = size - off;
1519
1520 if (msc->mode == MSC_MODE_SINGLE) {
1521 ret = msc_single_to_user(msc, buf, off, len);
1522 if (ret >= 0)
1523 *ppos += ret;
1524 } else if (msc->mode == MSC_MODE_MULTI) {
1525 struct msc_win_to_user_struct u = {
1526 .buf = buf,
1527 .offset = 0,
1528 };
1529
1530 ret = msc_buffer_iterate(iter, len, &u, msc_win_to_user);
1531 if (ret >= 0)
1532 *ppos = iter->offset;
1533 } else {
1534 ret = -EINVAL;
1535 }
1536
1537 put_count:
1538 atomic_dec(&msc->user_count);
1539
1540 return ret;
1541 }
1542
1543
1544
1545
1546
1547 static void msc_mmap_open(struct vm_area_struct *vma)
1548 {
1549 struct msc_iter *iter = vma->vm_file->private_data;
1550 struct msc *msc = iter->msc;
1551
1552 atomic_inc(&msc->mmap_count);
1553 }
1554
1555 static void msc_mmap_close(struct vm_area_struct *vma)
1556 {
1557 struct msc_iter *iter = vma->vm_file->private_data;
1558 struct msc *msc = iter->msc;
1559 unsigned long pg;
1560
1561 if (!atomic_dec_and_mutex_lock(&msc->mmap_count, &msc->buf_mutex))
1562 return;
1563
1564
1565 for (pg = 0; pg < msc->nr_pages; pg++) {
1566 struct page *page = msc_buffer_get_page(msc, pg);
1567
1568 if (WARN_ON_ONCE(!page))
1569 continue;
1570
1571 if (page->mapping)
1572 page->mapping = NULL;
1573 }
1574
1575
1576 atomic_dec(&msc->user_count);
1577 mutex_unlock(&msc->buf_mutex);
1578 }
1579
1580 static vm_fault_t msc_mmap_fault(struct vm_fault *vmf)
1581 {
1582 struct msc_iter *iter = vmf->vma->vm_file->private_data;
1583 struct msc *msc = iter->msc;
1584
1585 vmf->page = msc_buffer_get_page(msc, vmf->pgoff);
1586 if (!vmf->page)
1587 return VM_FAULT_SIGBUS;
1588
1589 get_page(vmf->page);
1590 vmf->page->mapping = vmf->vma->vm_file->f_mapping;
1591 vmf->page->index = vmf->pgoff;
1592
1593 return 0;
1594 }
1595
1596 static const struct vm_operations_struct msc_mmap_ops = {
1597 .open = msc_mmap_open,
1598 .close = msc_mmap_close,
1599 .fault = msc_mmap_fault,
1600 };
1601
1602 static int intel_th_msc_mmap(struct file *file, struct vm_area_struct *vma)
1603 {
1604 unsigned long size = vma->vm_end - vma->vm_start;
1605 struct msc_iter *iter = vma->vm_file->private_data;
1606 struct msc *msc = iter->msc;
1607 int ret = -EINVAL;
1608
1609 if (!size || offset_in_page(size))
1610 return -EINVAL;
1611
1612 if (vma->vm_pgoff)
1613 return -EINVAL;
1614
1615
1616 if (!atomic_inc_unless_negative(&msc->user_count))
1617 return -EINVAL;
1618
1619 if (msc->mode != MSC_MODE_SINGLE &&
1620 msc->mode != MSC_MODE_MULTI)
1621 goto out;
1622
1623 if (size >> PAGE_SHIFT != msc->nr_pages)
1624 goto out;
1625
1626 atomic_set(&msc->mmap_count, 1);
1627 ret = 0;
1628
1629 out:
1630 if (ret)
1631 atomic_dec(&msc->user_count);
1632
1633 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1634 vma->vm_flags |= VM_DONTEXPAND | VM_DONTCOPY;
1635 vma->vm_ops = &msc_mmap_ops;
1636 return ret;
1637 }
1638
1639 static const struct file_operations intel_th_msc_fops = {
1640 .open = intel_th_msc_open,
1641 .release = intel_th_msc_release,
1642 .read = intel_th_msc_read,
1643 .mmap = intel_th_msc_mmap,
1644 .llseek = no_llseek,
1645 .owner = THIS_MODULE,
1646 };
1647
1648 static void intel_th_msc_wait_empty(struct intel_th_device *thdev)
1649 {
1650 struct msc *msc = dev_get_drvdata(&thdev->dev);
1651 unsigned long count;
1652 u32 reg;
1653
1654 for (reg = 0, count = MSC_PLE_WAITLOOP_DEPTH;
1655 count && !(reg & MSCSTS_PLE); count--) {
1656 reg = __raw_readl(msc->reg_base + REG_MSU_MSC0STS);
1657 cpu_relax();
1658 }
1659
1660 if (!count)
1661 dev_dbg(msc_dev(msc), "timeout waiting for MSC0 PLE\n");
1662 }
1663
1664 static int intel_th_msc_init(struct msc *msc)
1665 {
1666 atomic_set(&msc->user_count, -1);
1667
1668 msc->mode = MSC_MODE_MULTI;
1669 mutex_init(&msc->buf_mutex);
1670 INIT_LIST_HEAD(&msc->win_list);
1671 INIT_LIST_HEAD(&msc->iter_list);
1672
1673 msc->burst_len =
1674 (ioread32(msc->reg_base + REG_MSU_MSC0CTL) & MSC_LEN) >>
1675 __ffs(MSC_LEN);
1676
1677 return 0;
1678 }
1679
1680 static int msc_win_switch(struct msc *msc)
1681 {
1682 struct msc_window *first;
1683
1684 if (list_empty(&msc->win_list))
1685 return -EINVAL;
1686
1687 first = list_first_entry(&msc->win_list, struct msc_window, entry);
1688
1689 if (msc_is_last_win(msc->cur_win))
1690 msc->cur_win = first;
1691 else
1692 msc->cur_win = list_next_entry(msc->cur_win, entry);
1693
1694 msc->base = msc_win_base(msc->cur_win);
1695 msc->base_addr = msc_win_base_dma(msc->cur_win);
1696
1697 intel_th_trace_switch(msc->thdev);
1698
1699 return 0;
1700 }
1701
1702
1703
1704
1705
1706
1707 void intel_th_msc_window_unlock(struct device *dev, struct sg_table *sgt)
1708 {
1709 struct msc *msc = dev_get_drvdata(dev);
1710 struct msc_window *win;
1711
1712 if (!sgt)
1713 return;
1714
1715 win = msc_find_window(msc, sgt, false);
1716 if (!win)
1717 return;
1718
1719 msc_win_set_lockout(win, WIN_LOCKED, WIN_READY);
1720 }
1721 EXPORT_SYMBOL_GPL(intel_th_msc_window_unlock);
1722
1723 static void msc_work(struct work_struct *work)
1724 {
1725 struct msc *msc = container_of(work, struct msc, work);
1726
1727 intel_th_msc_deactivate(msc->thdev);
1728 }
1729
1730 static irqreturn_t intel_th_msc_interrupt(struct intel_th_device *thdev)
1731 {
1732 struct msc *msc = dev_get_drvdata(&thdev->dev);
1733 u32 msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS);
1734 u32 mask = msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST;
1735 struct msc_window *win, *next_win;
1736
1737 if (!msc->do_irq || !msc->mbuf)
1738 return IRQ_NONE;
1739
1740 msusts &= mask;
1741
1742 if (!msusts)
1743 return msc->enabled ? IRQ_HANDLED : IRQ_NONE;
1744
1745 iowrite32(msusts, msc->msu_base + REG_MSU_MSUSTS);
1746
1747 if (!msc->enabled)
1748 return IRQ_NONE;
1749
1750
1751 win = msc->cur_win;
1752 if (!win)
1753 return IRQ_HANDLED;
1754 next_win = msc_next_window(win);
1755 if (!next_win)
1756 return IRQ_HANDLED;
1757
1758
1759 if (msc_win_set_lockout(next_win, WIN_READY, WIN_INUSE)) {
1760 schedule_work(&msc->work);
1761 return IRQ_HANDLED;
1762 }
1763
1764
1765 msc_win_set_lockout(win, WIN_INUSE, WIN_LOCKED);
1766
1767 msc_win_switch(msc);
1768
1769 if (msc->mbuf && msc->mbuf->ready)
1770 msc->mbuf->ready(msc->mbuf_priv, win->sgt,
1771 msc_win_total_sz(win));
1772
1773 return IRQ_HANDLED;
1774 }
1775
1776 static const char * const msc_mode[] = {
1777 [MSC_MODE_SINGLE] = "single",
1778 [MSC_MODE_MULTI] = "multi",
1779 [MSC_MODE_EXI] = "ExI",
1780 [MSC_MODE_DEBUG] = "debug",
1781 };
1782
1783 static ssize_t
1784 wrap_show(struct device *dev, struct device_attribute *attr, char *buf)
1785 {
1786 struct msc *msc = dev_get_drvdata(dev);
1787
1788 return scnprintf(buf, PAGE_SIZE, "%d\n", msc->wrap);
1789 }
1790
1791 static ssize_t
1792 wrap_store(struct device *dev, struct device_attribute *attr, const char *buf,
1793 size_t size)
1794 {
1795 struct msc *msc = dev_get_drvdata(dev);
1796 unsigned long val;
1797 int ret;
1798
1799 ret = kstrtoul(buf, 10, &val);
1800 if (ret)
1801 return ret;
1802
1803 msc->wrap = !!val;
1804
1805 return size;
1806 }
1807
1808 static DEVICE_ATTR_RW(wrap);
1809
1810 static void msc_buffer_unassign(struct msc *msc)
1811 {
1812 lockdep_assert_held(&msc->buf_mutex);
1813
1814 if (!msc->mbuf)
1815 return;
1816
1817 msc->mbuf->unassign(msc->mbuf_priv);
1818 msu_buffer_put(msc->mbuf);
1819 msc->mbuf_priv = NULL;
1820 msc->mbuf = NULL;
1821 }
1822
1823 static ssize_t
1824 mode_show(struct device *dev, struct device_attribute *attr, char *buf)
1825 {
1826 struct msc *msc = dev_get_drvdata(dev);
1827 const char *mode = msc_mode[msc->mode];
1828 ssize_t ret;
1829
1830 mutex_lock(&msc->buf_mutex);
1831 if (msc->mbuf)
1832 mode = msc->mbuf->name;
1833 ret = scnprintf(buf, PAGE_SIZE, "%s\n", mode);
1834 mutex_unlock(&msc->buf_mutex);
1835
1836 return ret;
1837 }
1838
1839 static ssize_t
1840 mode_store(struct device *dev, struct device_attribute *attr, const char *buf,
1841 size_t size)
1842 {
1843 const struct msu_buffer *mbuf = NULL;
1844 struct msc *msc = dev_get_drvdata(dev);
1845 size_t len = size;
1846 char *cp, *mode;
1847 int i, ret;
1848
1849 if (!capable(CAP_SYS_RAWIO))
1850 return -EPERM;
1851
1852 cp = memchr(buf, '\n', len);
1853 if (cp)
1854 len = cp - buf;
1855
1856 mode = kstrndup(buf, len, GFP_KERNEL);
1857 if (!mode)
1858 return -ENOMEM;
1859
1860 i = match_string(msc_mode, ARRAY_SIZE(msc_mode), mode);
1861 if (i >= 0) {
1862 kfree(mode);
1863 goto found;
1864 }
1865
1866
1867 if (!msc->do_irq) {
1868 kfree(mode);
1869 return -EINVAL;
1870 }
1871
1872 mbuf = msu_buffer_get(mode);
1873 kfree(mode);
1874 if (mbuf)
1875 goto found;
1876
1877 return -EINVAL;
1878
1879 found:
1880 mutex_lock(&msc->buf_mutex);
1881 ret = 0;
1882
1883
1884 if (mbuf && mbuf == msc->mbuf) {
1885
1886 msu_buffer_put(mbuf);
1887 goto unlock;
1888 }
1889
1890 ret = msc_buffer_unlocked_free_unless_used(msc);
1891 if (ret)
1892 goto unlock;
1893
1894 if (mbuf) {
1895 void *mbuf_priv = mbuf->assign(dev, &i);
1896
1897 if (!mbuf_priv) {
1898 ret = -ENOMEM;
1899 goto unlock;
1900 }
1901
1902 msc_buffer_unassign(msc);
1903 msc->mbuf_priv = mbuf_priv;
1904 msc->mbuf = mbuf;
1905 } else {
1906 msc_buffer_unassign(msc);
1907 }
1908
1909 msc->mode = i;
1910
1911 unlock:
1912 if (ret && mbuf)
1913 msu_buffer_put(mbuf);
1914 mutex_unlock(&msc->buf_mutex);
1915
1916 return ret ? ret : size;
1917 }
1918
1919 static DEVICE_ATTR_RW(mode);
1920
1921 static ssize_t
1922 nr_pages_show(struct device *dev, struct device_attribute *attr, char *buf)
1923 {
1924 struct msc *msc = dev_get_drvdata(dev);
1925 struct msc_window *win;
1926 size_t count = 0;
1927
1928 mutex_lock(&msc->buf_mutex);
1929
1930 if (msc->mode == MSC_MODE_SINGLE)
1931 count = scnprintf(buf, PAGE_SIZE, "%ld\n", msc->nr_pages);
1932 else if (msc->mode == MSC_MODE_MULTI) {
1933 list_for_each_entry(win, &msc->win_list, entry) {
1934 count += scnprintf(buf + count, PAGE_SIZE - count,
1935 "%d%c", win->nr_blocks,
1936 msc_is_last_win(win) ? '\n' : ',');
1937 }
1938 } else {
1939 count = scnprintf(buf, PAGE_SIZE, "unsupported\n");
1940 }
1941
1942 mutex_unlock(&msc->buf_mutex);
1943
1944 return count;
1945 }
1946
1947 static ssize_t
1948 nr_pages_store(struct device *dev, struct device_attribute *attr,
1949 const char *buf, size_t size)
1950 {
1951 struct msc *msc = dev_get_drvdata(dev);
1952 unsigned long val, *win = NULL, *rewin;
1953 size_t len = size;
1954 const char *p = buf;
1955 char *end, *s;
1956 int ret, nr_wins = 0;
1957
1958 if (!capable(CAP_SYS_RAWIO))
1959 return -EPERM;
1960
1961 ret = msc_buffer_free_unless_used(msc);
1962 if (ret)
1963 return ret;
1964
1965
1966 end = memchr(buf, '\n', len);
1967 if (end)
1968 len = end - buf;
1969
1970 do {
1971 end = memchr(p, ',', len);
1972 s = kstrndup(p, end ? end - p : len, GFP_KERNEL);
1973 if (!s) {
1974 ret = -ENOMEM;
1975 goto free_win;
1976 }
1977
1978 ret = kstrtoul(s, 10, &val);
1979 kfree(s);
1980
1981 if (ret || !val)
1982 goto free_win;
1983
1984 if (nr_wins && msc->mode == MSC_MODE_SINGLE) {
1985 ret = -EINVAL;
1986 goto free_win;
1987 }
1988
1989 nr_wins++;
1990 rewin = krealloc(win, sizeof(*win) * nr_wins, GFP_KERNEL);
1991 if (!rewin) {
1992 kfree(win);
1993 return -ENOMEM;
1994 }
1995
1996 win = rewin;
1997 win[nr_wins - 1] = val;
1998
1999 if (!end)
2000 break;
2001
2002
2003 len -= end - p + 1;
2004 p = end + 1;
2005 } while (len);
2006
2007 mutex_lock(&msc->buf_mutex);
2008 ret = msc_buffer_alloc(msc, win, nr_wins);
2009 mutex_unlock(&msc->buf_mutex);
2010
2011 free_win:
2012 kfree(win);
2013
2014 return ret ? ret : size;
2015 }
2016
2017 static DEVICE_ATTR_RW(nr_pages);
2018
2019 static ssize_t
2020 win_switch_store(struct device *dev, struct device_attribute *attr,
2021 const char *buf, size_t size)
2022 {
2023 struct msc *msc = dev_get_drvdata(dev);
2024 unsigned long val;
2025 int ret;
2026
2027 ret = kstrtoul(buf, 10, &val);
2028 if (ret)
2029 return ret;
2030
2031 if (val != 1)
2032 return -EINVAL;
2033
2034 ret = -EINVAL;
2035 mutex_lock(&msc->buf_mutex);
2036
2037
2038
2039
2040
2041 if (msc->mode == MSC_MODE_MULTI && !msc->mbuf)
2042 ret = msc_win_switch(msc);
2043 mutex_unlock(&msc->buf_mutex);
2044
2045 return ret ? ret : size;
2046 }
2047
2048 static DEVICE_ATTR_WO(win_switch);
2049
2050 static struct attribute *msc_output_attrs[] = {
2051 &dev_attr_wrap.attr,
2052 &dev_attr_mode.attr,
2053 &dev_attr_nr_pages.attr,
2054 &dev_attr_win_switch.attr,
2055 NULL,
2056 };
2057
2058 static struct attribute_group msc_output_group = {
2059 .attrs = msc_output_attrs,
2060 };
2061
2062 static int intel_th_msc_probe(struct intel_th_device *thdev)
2063 {
2064 struct device *dev = &thdev->dev;
2065 struct resource *res;
2066 struct msc *msc;
2067 void __iomem *base;
2068 int err;
2069
2070 res = intel_th_device_get_resource(thdev, IORESOURCE_MEM, 0);
2071 if (!res)
2072 return -ENODEV;
2073
2074 base = devm_ioremap(dev, res->start, resource_size(res));
2075 if (!base)
2076 return -ENOMEM;
2077
2078 msc = devm_kzalloc(dev, sizeof(*msc), GFP_KERNEL);
2079 if (!msc)
2080 return -ENOMEM;
2081
2082 res = intel_th_device_get_resource(thdev, IORESOURCE_IRQ, 1);
2083 if (!res)
2084 msc->do_irq = 1;
2085
2086 msc->index = thdev->id;
2087
2088 msc->thdev = thdev;
2089 msc->reg_base = base + msc->index * 0x100;
2090 msc->msu_base = base;
2091
2092 INIT_WORK(&msc->work, msc_work);
2093 err = intel_th_msc_init(msc);
2094 if (err)
2095 return err;
2096
2097 dev_set_drvdata(dev, msc);
2098
2099 return 0;
2100 }
2101
2102 static void intel_th_msc_remove(struct intel_th_device *thdev)
2103 {
2104 struct msc *msc = dev_get_drvdata(&thdev->dev);
2105 int ret;
2106
2107 intel_th_msc_deactivate(thdev);
2108
2109
2110
2111
2112
2113
2114 ret = msc_buffer_free_unless_used(msc);
2115 WARN_ON_ONCE(ret);
2116 }
2117
2118 static struct intel_th_driver intel_th_msc_driver = {
2119 .probe = intel_th_msc_probe,
2120 .remove = intel_th_msc_remove,
2121 .irq = intel_th_msc_interrupt,
2122 .wait_empty = intel_th_msc_wait_empty,
2123 .activate = intel_th_msc_activate,
2124 .deactivate = intel_th_msc_deactivate,
2125 .fops = &intel_th_msc_fops,
2126 .attr_group = &msc_output_group,
2127 .driver = {
2128 .name = "msc",
2129 .owner = THIS_MODULE,
2130 },
2131 };
2132
2133 module_driver(intel_th_msc_driver,
2134 intel_th_driver_register,
2135 intel_th_driver_unregister);
2136
2137 MODULE_LICENSE("GPL v2");
2138 MODULE_DESCRIPTION("Intel(R) Trace Hub Memory Storage Unit driver");
2139 MODULE_AUTHOR("Alexander Shishkin <alexander.shishkin@linux.intel.com>");