1/*
2  This file is provided under a dual BSD/GPLv2 license.  When using or
3  redistributing this file, you may do so under either license.
4
5  GPL LICENSE SUMMARY
6  Copyright(c) 2014 Intel Corporation.
7  This program is free software; you can redistribute it and/or modify
8  it under the terms of version 2 of the GNU General Public License as
9  published by the Free Software Foundation.
10
11  This program is distributed in the hope that it will be useful, but
12  WITHOUT ANY WARRANTY; without even the implied warranty of
13  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  General Public License for more details.
15
16  Contact Information:
17  qat-linux@intel.com
18
19  BSD LICENSE
20  Copyright(c) 2014 Intel Corporation.
21  Redistribution and use in source and binary forms, with or without
22  modification, are permitted provided that the following conditions
23  are met:
24
25    * Redistributions of source code must retain the above copyright
26      notice, this list of conditions and the following disclaimer.
27    * Redistributions in binary form must reproduce the above copyright
28      notice, this list of conditions and the following disclaimer in
29      the documentation and/or other materials provided with the
30      distribution.
31    * Neither the name of Intel Corporation nor the names of its
32      contributors may be used to endorse or promote products derived
33      from this software without specific prior written permission.
34
35  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46*/
47#include <linux/kernel.h>
48#include <linux/init.h>
49#include <linux/types.h>
50#include <linux/pci.h>
51#include <linux/slab.h>
52#include <linux/errno.h>
53#include <linux/interrupt.h>
54#include <adf_accel_devices.h>
55#include <adf_common_drv.h>
56#include <adf_cfg.h>
57#include <adf_cfg_strings.h>
58#include <adf_cfg_common.h>
59#include <adf_transport_access_macros.h>
60#include <adf_transport_internal.h>
61#include "adf_drv.h"
62#include "adf_dh895xcc_hw_data.h"
63
64static int adf_enable_msix(struct adf_accel_dev *accel_dev)
65{
66	struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
67	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
68	u32 msix_num_entries = 1;
69
70	/* If SR-IOV is disabled, add entries for each bank */
71	if (!accel_dev->pf.vf_info) {
72		int i;
73
74		msix_num_entries += hw_data->num_banks;
75		for (i = 0; i < msix_num_entries; i++)
76			pci_dev_info->msix_entries.entries[i].entry = i;
77	} else {
78		pci_dev_info->msix_entries.entries[0].entry =
79			hw_data->num_banks;
80	}
81
82	if (pci_enable_msix_exact(pci_dev_info->pci_dev,
83				  pci_dev_info->msix_entries.entries,
84				  msix_num_entries)) {
85		dev_err(&GET_DEV(accel_dev), "Failed to enable MSI-X IRQ(s)\n");
86		return -EFAULT;
87	}
88	return 0;
89}
90
91static void adf_disable_msix(struct adf_accel_pci *pci_dev_info)
92{
93	pci_disable_msix(pci_dev_info->pci_dev);
94}
95
96static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
97{
98	struct adf_etr_bank_data *bank = bank_ptr;
99
100	WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0);
101	tasklet_hi_schedule(&bank->resp_handler);
102	return IRQ_HANDLED;
103}
104
105static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
106{
107	struct adf_accel_dev *accel_dev = dev_ptr;
108
109#ifdef CONFIG_PCI_IOV
110	/* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */
111	if (accel_dev->pf.vf_info) {
112		void __iomem *pmisc_bar_addr =
113		    (&GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR])->virt_addr;
114		u32 vf_mask;
115
116		/* Get the interrupt sources triggered by VFs */
117		vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCC_ERRSOU5) &
118			    0x0000FFFF) << 16) |
119			  ((ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCC_ERRSOU3) &
120			    0x01FFFE00) >> 9);
121
122		if (vf_mask) {
123			struct adf_accel_vf_info *vf_info;
124			bool irq_handled = false;
125			int i;
126
127			/* Disable VF2PF interrupts for VFs with pending ints */
128			adf_disable_vf2pf_interrupts(accel_dev, vf_mask);
129
130			/*
131			 * Schedule tasklets to handle VF2PF interrupt BHs
132			 * unless the VF is malicious and is attempting to
133			 * flood the host OS with VF2PF interrupts.
134			 */
135			for_each_set_bit(i, (const unsigned long *)&vf_mask,
136					 (sizeof(vf_mask) * BITS_PER_BYTE)) {
137				vf_info = accel_dev->pf.vf_info + i;
138
139				if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
140					dev_info(&GET_DEV(accel_dev),
141						 "Too many ints from VF%d\n",
142						  vf_info->vf_nr + 1);
143					continue;
144				}
145
146				/* Tasklet will re-enable ints from this VF */
147				tasklet_hi_schedule(&vf_info->vf2pf_bh_tasklet);
148				irq_handled = true;
149			}
150
151			if (irq_handled)
152				return IRQ_HANDLED;
153		}
154	}
155#endif /* CONFIG_PCI_IOV */
156
157	dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",
158		accel_dev->accel_id);
159
160	return IRQ_NONE;
161}
162
163static int adf_request_irqs(struct adf_accel_dev *accel_dev)
164{
165	struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
166	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
167	struct msix_entry *msixe = pci_dev_info->msix_entries.entries;
168	struct adf_etr_data *etr_data = accel_dev->transport;
169	int ret, i = 0;
170	char *name;
171
172	/* Request msix irq for all banks unless SR-IOV enabled */
173	if (!accel_dev->pf.vf_info) {
174		for (i = 0; i < hw_data->num_banks; i++) {
175			struct adf_etr_bank_data *bank = &etr_data->banks[i];
176			unsigned int cpu, cpus = num_online_cpus();
177
178			name = *(pci_dev_info->msix_entries.names + i);
179			snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
180				 "qat%d-bundle%d", accel_dev->accel_id, i);
181			ret = request_irq(msixe[i].vector,
182					  adf_msix_isr_bundle, 0, name, bank);
183			if (ret) {
184				dev_err(&GET_DEV(accel_dev),
185					"failed to enable irq %d for %s\n",
186					msixe[i].vector, name);
187				return ret;
188			}
189
190			cpu = ((accel_dev->accel_id * hw_data->num_banks) +
191			       i) % cpus;
192			irq_set_affinity_hint(msixe[i].vector,
193					      get_cpu_mask(cpu));
194		}
195	}
196
197	/* Request msix irq for AE */
198	name = *(pci_dev_info->msix_entries.names + i);
199	snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
200		 "qat%d-ae-cluster", accel_dev->accel_id);
201	ret = request_irq(msixe[i].vector, adf_msix_isr_ae, 0, name, accel_dev);
202	if (ret) {
203		dev_err(&GET_DEV(accel_dev),
204			"failed to enable irq %d, for %s\n",
205			msixe[i].vector, name);
206		return ret;
207	}
208	return ret;
209}
210
211static void adf_free_irqs(struct adf_accel_dev *accel_dev)
212{
213	struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
214	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
215	struct msix_entry *msixe = pci_dev_info->msix_entries.entries;
216	struct adf_etr_data *etr_data = accel_dev->transport;
217	int i = 0;
218
219	if (pci_dev_info->msix_entries.num_entries > 1) {
220		for (i = 0; i < hw_data->num_banks; i++) {
221			irq_set_affinity_hint(msixe[i].vector, NULL);
222			free_irq(msixe[i].vector, &etr_data->banks[i]);
223		}
224	}
225	irq_set_affinity_hint(msixe[i].vector, NULL);
226	free_irq(msixe[i].vector, accel_dev);
227}
228
229static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev)
230{
231	int i;
232	char **names;
233	struct msix_entry *entries;
234	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
235	u32 msix_num_entries = 1;
236
237	/* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */
238	if (!accel_dev->pf.vf_info)
239		msix_num_entries += hw_data->num_banks;
240
241	entries = kzalloc_node(msix_num_entries * sizeof(*entries),
242			       GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev)));
243	if (!entries)
244		return -ENOMEM;
245
246	names = kcalloc(msix_num_entries, sizeof(char *), GFP_KERNEL);
247	if (!names) {
248		kfree(entries);
249		return -ENOMEM;
250	}
251	for (i = 0; i < msix_num_entries; i++) {
252		*(names + i) = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL);
253		if (!(*(names + i)))
254			goto err;
255	}
256	accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries;
257	accel_dev->accel_pci_dev.msix_entries.entries = entries;
258	accel_dev->accel_pci_dev.msix_entries.names = names;
259	return 0;
260err:
261	for (i = 0; i < msix_num_entries; i++)
262		kfree(*(names + i));
263	kfree(entries);
264	kfree(names);
265	return -ENOMEM;
266}
267
268static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev)
269{
270	char **names = accel_dev->accel_pci_dev.msix_entries.names;
271	int i;
272
273	kfree(accel_dev->accel_pci_dev.msix_entries.entries);
274	for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++)
275		kfree(*(names + i));
276	kfree(names);
277}
278
279static int adf_setup_bh(struct adf_accel_dev *accel_dev)
280{
281	struct adf_etr_data *priv_data = accel_dev->transport;
282	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
283	int i;
284
285	for (i = 0; i < hw_data->num_banks; i++)
286		tasklet_init(&priv_data->banks[i].resp_handler,
287			     adf_response_handler,
288			     (unsigned long)&priv_data->banks[i]);
289	return 0;
290}
291
292static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
293{
294	struct adf_etr_data *priv_data = accel_dev->transport;
295	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
296	int i;
297
298	for (i = 0; i < hw_data->num_banks; i++) {
299		tasklet_disable(&priv_data->banks[i].resp_handler);
300		tasklet_kill(&priv_data->banks[i].resp_handler);
301	}
302}
303
304void adf_isr_resource_free(struct adf_accel_dev *accel_dev)
305{
306	adf_free_irqs(accel_dev);
307	adf_cleanup_bh(accel_dev);
308	adf_disable_msix(&accel_dev->accel_pci_dev);
309	adf_isr_free_msix_entry_table(accel_dev);
310}
311
312int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
313{
314	int ret;
315
316	ret = adf_isr_alloc_msix_entry_table(accel_dev);
317	if (ret)
318		return ret;
319	if (adf_enable_msix(accel_dev))
320		goto err_out;
321
322	if (adf_setup_bh(accel_dev))
323		goto err_out;
324
325	if (adf_request_irqs(accel_dev))
326		goto err_out;
327
328	return 0;
329err_out:
330	adf_isr_resource_free(accel_dev);
331	return -EFAULT;
332}
333