1/* 2 This file is provided under a dual BSD/GPLv2 license. When using or 3 redistributing this file, you may do so under either license. 4 5 GPL LICENSE SUMMARY 6 Copyright(c) 2014 Intel Corporation. 7 This program is free software; you can redistribute it and/or modify 8 it under the terms of version 2 of the GNU General Public License as 9 published by the Free Software Foundation. 10 11 This program is distributed in the hope that it will be useful, but 12 WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 General Public License for more details. 15 16 Contact Information: 17 qat-linux@intel.com 18 19 BSD LICENSE 20 Copyright(c) 2014 Intel Corporation. 21 Redistribution and use in source and binary forms, with or without 22 modification, are permitted provided that the following conditions 23 are met: 24 25 * Redistributions of source code must retain the above copyright 26 notice, this list of conditions and the following disclaimer. 27 * Redistributions in binary form must reproduce the above copyright 28 notice, this list of conditions and the following disclaimer in 29 the documentation and/or other materials provided with the 30 distribution. 31 * Neither the name of Intel Corporation nor the names of its 32 contributors may be used to endorse or promote products derived 33 from this software without specific prior written permission. 34 35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46*/ 47#include <linux/kernel.h> 48#include <linux/module.h> 49#include <linux/pci.h> 50#include <linux/init.h> 51#include <linux/types.h> 52#include <linux/fs.h> 53#include <linux/slab.h> 54#include <linux/errno.h> 55#include <linux/device.h> 56#include <linux/dma-mapping.h> 57#include <linux/platform_device.h> 58#include <linux/workqueue.h> 59#include <linux/io.h> 60#include <adf_accel_devices.h> 61#include <adf_common_drv.h> 62#include <adf_cfg.h> 63#include <adf_transport_access_macros.h> 64#include "adf_dh895xccvf_hw_data.h" 65#include "adf_drv.h" 66 67static const char adf_driver_name[] = ADF_DH895XCCVF_DEVICE_NAME; 68 69#define ADF_SYSTEM_DEVICE(device_id) \ 70 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} 71 72static const struct pci_device_id adf_pci_tbl[] = { 73 ADF_SYSTEM_DEVICE(ADF_DH895XCCIOV_PCI_DEVICE_ID), 74 {0,} 75}; 76MODULE_DEVICE_TABLE(pci, adf_pci_tbl); 77 78static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); 79static void adf_remove(struct pci_dev *dev); 80 81static struct pci_driver adf_driver = { 82 .id_table = adf_pci_tbl, 83 .name = adf_driver_name, 84 .probe = adf_probe, 85 .remove = adf_remove, 86}; 87 88static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) 89{ 90 pci_release_regions(accel_dev->accel_pci_dev.pci_dev); 91 pci_disable_device(accel_dev->accel_pci_dev.pci_dev); 92} 93 94static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) 95{ 96 struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; 97 struct adf_accel_dev *pf; 98 int i; 99 100 for (i = 0; i < ADF_PCI_MAX_BARS; i++) { 101 struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; 102 103 if (bar->virt_addr) 104 pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); 105 } 106 107 if (accel_dev->hw_device) { 108 switch (accel_pci_dev->pci_dev->device) { 109 case ADF_DH895XCCIOV_PCI_DEVICE_ID: 110 adf_clean_hw_data_dh895xcciov(accel_dev->hw_device); 111 break; 112 default: 113 break; 114 } 115 kfree(accel_dev->hw_device); 116 accel_dev->hw_device = NULL; 117 } 118 adf_cfg_dev_remove(accel_dev); 119 debugfs_remove(accel_dev->debugfs_dir); 120 pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); 121 adf_devmgr_rm_dev(accel_dev, pf); 122} 123 124static int adf_dev_configure(struct adf_accel_dev *accel_dev) 125{ 126 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 127 unsigned long val, bank = 0; 128 129 if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) 130 goto err; 131 if (adf_cfg_section_add(accel_dev, "Accelerator0")) 132 goto err; 133 134 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, 0); 135 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, 136 (void *)&bank, ADF_DEC)) 137 goto err; 138 139 val = bank; 140 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, 0); 141 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, 142 (void *)&val, ADF_DEC)) 143 goto err; 144 145 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, 0); 146 147 val = 128; 148 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, 149 (void *)&val, ADF_DEC)) 150 goto err; 151 152 val = 512; 153 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, 0); 154 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 155 key, (void *)&val, ADF_DEC)) 156 goto err; 157 158 val = 0; 159 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, 0); 160 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 161 key, (void *)&val, ADF_DEC)) 162 goto err; 163 164 val = 2; 165 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, 0); 166 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 167 key, (void *)&val, ADF_DEC)) 168 goto err; 169 170 val = 8; 171 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, 0); 172 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 173 key, (void *)&val, ADF_DEC)) 174 goto err; 175 176 val = 10; 177 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, 0); 178 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 179 key, (void *)&val, ADF_DEC)) 180 goto err; 181 182 val = ADF_COALESCING_DEF_TIME; 183 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, 184 (int)bank); 185 if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", 186 key, (void *)&val, ADF_DEC)) 187 goto err; 188 189 val = 1; 190 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 191 ADF_NUM_CY, (void *)&val, ADF_DEC)) 192 goto err; 193 194 set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); 195 return 0; 196err: 197 dev_err(&GET_DEV(accel_dev), "Failed to configure QAT accel dev\n"); 198 return -EINVAL; 199} 200 201static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 202{ 203 struct adf_accel_dev *accel_dev; 204 struct adf_accel_dev *pf; 205 struct adf_accel_pci *accel_pci_dev; 206 struct adf_hw_device_data *hw_data; 207 char name[ADF_DEVICE_NAME_LENGTH]; 208 unsigned int i, bar_nr; 209 int ret, bar_mask; 210 211 switch (ent->device) { 212 case ADF_DH895XCCIOV_PCI_DEVICE_ID: 213 break; 214 default: 215 dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); 216 return -ENODEV; 217 } 218 219 accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, 220 dev_to_node(&pdev->dev)); 221 if (!accel_dev) 222 return -ENOMEM; 223 224 accel_dev->is_vf = true; 225 pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); 226 accel_pci_dev = &accel_dev->accel_pci_dev; 227 accel_pci_dev->pci_dev = pdev; 228 229 /* Add accel device to accel table */ 230 if (adf_devmgr_add_dev(accel_dev, pf)) { 231 dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); 232 kfree(accel_dev); 233 return -EFAULT; 234 } 235 INIT_LIST_HEAD(&accel_dev->crypto_list); 236 237 accel_dev->owner = THIS_MODULE; 238 /* Allocate and configure device configuration structure */ 239 hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, 240 dev_to_node(&pdev->dev)); 241 if (!hw_data) { 242 ret = -ENOMEM; 243 goto out_err; 244 } 245 accel_dev->hw_device = hw_data; 246 switch (ent->device) { 247 case ADF_DH895XCCIOV_PCI_DEVICE_ID: 248 adf_init_hw_data_dh895xcciov(accel_dev->hw_device); 249 break; 250 default: 251 ret = -ENODEV; 252 goto out_err; 253 } 254 255 /* Get Accelerators and Accelerators Engines masks */ 256 hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); 257 hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); 258 accel_pci_dev->sku = hw_data->get_sku(hw_data); 259 260 /* Create dev top level debugfs entry */ 261 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", 262 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, 263 pdev->bus->number, PCI_SLOT(pdev->devfn), 264 PCI_FUNC(pdev->devfn)); 265 266 accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); 267 if (!accel_dev->debugfs_dir) { 268 dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name); 269 ret = -EINVAL; 270 goto out_err; 271 } 272 273 /* Create device configuration table */ 274 ret = adf_cfg_dev_add(accel_dev); 275 if (ret) 276 goto out_err; 277 278 /* enable PCI device */ 279 if (pci_enable_device(pdev)) { 280 ret = -EFAULT; 281 goto out_err; 282 } 283 284 /* set dma identifier */ 285 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 286 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { 287 dev_err(&pdev->dev, "No usable DMA configuration\n"); 288 ret = -EFAULT; 289 goto out_err_disable; 290 } else { 291 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 292 } 293 294 } else { 295 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 296 } 297 298 if (pci_request_regions(pdev, adf_driver_name)) { 299 ret = -EFAULT; 300 goto out_err_disable; 301 } 302 303 /* Find and map all the device's BARS */ 304 i = 0; 305 bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); 306 for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, 307 ADF_PCI_MAX_BARS * 2) { 308 struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; 309 310 bar->base_addr = pci_resource_start(pdev, bar_nr); 311 if (!bar->base_addr) 312 break; 313 bar->size = pci_resource_len(pdev, bar_nr); 314 bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); 315 if (!bar->virt_addr) { 316 dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); 317 ret = -EFAULT; 318 goto out_err_free_reg; 319 } 320 } 321 pci_set_master(pdev); 322 /* Completion for VF2PF request/response message exchange */ 323 init_completion(&accel_dev->vf.iov_msg_completion); 324 325 ret = adf_dev_configure(accel_dev); 326 if (ret) 327 goto out_err_free_reg; 328 329 ret = adf_dev_init(accel_dev); 330 if (ret) 331 goto out_err_dev_shutdown; 332 333 ret = adf_dev_start(accel_dev); 334 if (ret) 335 goto out_err_dev_stop; 336 337 return ret; 338 339out_err_dev_stop: 340 adf_dev_stop(accel_dev); 341out_err_dev_shutdown: 342 adf_dev_shutdown(accel_dev); 343out_err_free_reg: 344 pci_release_regions(accel_pci_dev->pci_dev); 345out_err_disable: 346 pci_disable_device(accel_pci_dev->pci_dev); 347out_err: 348 adf_cleanup_accel(accel_dev); 349 kfree(accel_dev); 350 return ret; 351} 352 353static void adf_remove(struct pci_dev *pdev) 354{ 355 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); 356 357 if (!accel_dev) { 358 pr_err("QAT: Driver removal failed\n"); 359 return; 360 } 361 if (adf_dev_stop(accel_dev)) 362 dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); 363 364 adf_dev_shutdown(accel_dev); 365 adf_cleanup_accel(accel_dev); 366 adf_cleanup_pci_dev(accel_dev); 367 kfree(accel_dev); 368} 369 370static int __init adfdrv_init(void) 371{ 372 request_module("intel_qat"); 373 374 if (pci_register_driver(&adf_driver)) { 375 pr_err("QAT: Driver initialization failed\n"); 376 return -EFAULT; 377 } 378 return 0; 379} 380 381static void __exit adfdrv_release(void) 382{ 383 pci_unregister_driver(&adf_driver); 384 adf_clean_vf_map(true); 385} 386 387module_init(adfdrv_init); 388module_exit(adfdrv_release); 389 390MODULE_LICENSE("Dual BSD/GPL"); 391MODULE_AUTHOR("Intel"); 392MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); 393MODULE_VERSION(ADF_DRV_VERSION); 394