1/* 2 This file is provided under a dual BSD/GPLv2 license. When using or 3 redistributing this file, you may do so under either license. 4 5 GPL LICENSE SUMMARY 6 Copyright(c) 2014 Intel Corporation. 7 This program is free software; you can redistribute it and/or modify 8 it under the terms of version 2 of the GNU General Public License as 9 published by the Free Software Foundation. 10 11 This program is distributed in the hope that it will be useful, but 12 WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 General Public License for more details. 15 16 Contact Information: 17 qat-linux@intel.com 18 19 BSD LICENSE 20 Copyright(c) 2014 Intel Corporation. 21 Redistribution and use in source and binary forms, with or without 22 modification, are permitted provided that the following conditions 23 are met: 24 25 * Redistributions of source code must retain the above copyright 26 notice, this list of conditions and the following disclaimer. 27 * Redistributions in binary form must reproduce the above copyright 28 notice, this list of conditions and the following disclaimer in 29 the documentation and/or other materials provided with the 30 distribution. 31 * Neither the name of Intel Corporation nor the names of its 32 contributors may be used to endorse or promote products derived 33 from this software without specific prior written permission. 34 35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46*/ 47#include <linux/kernel.h> 48#include <linux/module.h> 49#include <linux/pci.h> 50#include <linux/init.h> 51#include <linux/types.h> 52#include <linux/fs.h> 53#include <linux/slab.h> 54#include <linux/errno.h> 55#include <linux/device.h> 56#include <linux/dma-mapping.h> 57#include <linux/platform_device.h> 58#include <linux/workqueue.h> 59#include <linux/io.h> 60#include <adf_accel_devices.h> 61#include <adf_common_drv.h> 62#include <adf_cfg.h> 63#include <adf_transport_access_macros.h> 64#include "adf_dh895xcc_hw_data.h" 65#include "adf_drv.h" 66 67static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME; 68 69#define ADF_SYSTEM_DEVICE(device_id) \ 70 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} 71 72static const struct pci_device_id adf_pci_tbl[] = { 73 ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID), 74 {0,} 75}; 76MODULE_DEVICE_TABLE(pci, adf_pci_tbl); 77 78static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); 79static void adf_remove(struct pci_dev *dev); 80 81static struct pci_driver adf_driver = { 82 .id_table = adf_pci_tbl, 83 .name = adf_driver_name, 84 .probe = adf_probe, 85 .remove = adf_remove 86}; 87 88static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) 89{ 90 struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; 91 int i; 92 93 adf_dev_shutdown(accel_dev); 94 95 for (i = 0; i < ADF_PCI_MAX_BARS; i++) { 96 struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; 97 98 if (bar->virt_addr) 99 pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); 100 } 101 102 if (accel_dev->hw_device) { 103 switch (accel_dev->hw_device->pci_dev_id) { 104 case ADF_DH895XCC_PCI_DEVICE_ID: 105 adf_clean_hw_data_dh895xcc(accel_dev->hw_device); 106 break; 107 default: 108 break; 109 } 110 kfree(accel_dev->hw_device); 111 } 112 adf_cfg_dev_remove(accel_dev); 113 debugfs_remove(accel_dev->debugfs_dir); 114 adf_devmgr_rm_dev(accel_dev); 115 pci_release_regions(accel_pci_dev->pci_dev); 116 pci_disable_device(accel_pci_dev->pci_dev); 117 kfree(accel_dev); 118} 119 120static int adf_dev_configure(struct adf_accel_dev *accel_dev) 121{ 122 int cpus = num_online_cpus(); 123 int banks = GET_MAX_BANKS(accel_dev); 124 int instances = min(cpus, banks); 125 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 126 int i; 127 unsigned long val; 128 129 if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) 130 goto err; 131 if (adf_cfg_section_add(accel_dev, "Accelerator0")) 132 goto err; 133 for (i = 0; i < instances; i++) { 134 val = i; 135 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); 136 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 137 key, (void *)&val, ADF_DEC)) 138 goto err; 139 140 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, 141 i); 142 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 143 key, (void *)&val, ADF_DEC)) 144 goto err; 145 146 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); 147 val = 128; 148 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 149 key, (void *)&val, ADF_DEC)) 150 goto err; 151 152 val = 512; 153 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); 154 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 155 key, (void *)&val, ADF_DEC)) 156 goto err; 157 158 val = 0; 159 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); 160 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 161 key, (void *)&val, ADF_DEC)) 162 goto err; 163 164 val = 2; 165 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); 166 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 167 key, (void *)&val, ADF_DEC)) 168 goto err; 169 170 val = 4; 171 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i); 172 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 173 key, (void *)&val, ADF_DEC)) 174 goto err; 175 176 val = 8; 177 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); 178 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 179 key, (void *)&val, ADF_DEC)) 180 goto err; 181 182 val = 10; 183 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); 184 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 185 key, (void *)&val, ADF_DEC)) 186 goto err; 187 188 val = 12; 189 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i); 190 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 191 key, (void *)&val, ADF_DEC)) 192 goto err; 193 194 val = ADF_COALESCING_DEF_TIME; 195 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); 196 if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", 197 key, (void *)&val, ADF_DEC)) 198 goto err; 199 } 200 201 val = i; 202 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, 203 ADF_NUM_CY, (void *)&val, ADF_DEC)) 204 goto err; 205 206 set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); 207 return 0; 208err: 209 dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); 210 return -EINVAL; 211} 212 213static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 214{ 215 struct adf_accel_dev *accel_dev; 216 struct adf_accel_pci *accel_pci_dev; 217 struct adf_hw_device_data *hw_data; 218 char name[ADF_DEVICE_NAME_LENGTH]; 219 unsigned int i, bar_nr; 220 int ret; 221 222 switch (ent->device) { 223 case ADF_DH895XCC_PCI_DEVICE_ID: 224 break; 225 default: 226 dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); 227 return -ENODEV; 228 } 229 230 if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { 231 /* If the accelerator is connected to a node with no memory 232 * there is no point in using the accelerator since the remote 233 * memory transaction will be very slow. */ 234 dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); 235 return -EINVAL; 236 } 237 238 accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, 239 dev_to_node(&pdev->dev)); 240 if (!accel_dev) 241 return -ENOMEM; 242 243 INIT_LIST_HEAD(&accel_dev->crypto_list); 244 245 /* Add accel device to accel table. 246 * This should be called before adf_cleanup_accel is called */ 247 if (adf_devmgr_add_dev(accel_dev)) { 248 dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); 249 kfree(accel_dev); 250 return -EFAULT; 251 } 252 253 accel_dev->owner = THIS_MODULE; 254 /* Allocate and configure device configuration structure */ 255 hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, 256 dev_to_node(&pdev->dev)); 257 if (!hw_data) { 258 ret = -ENOMEM; 259 goto out_err; 260 } 261 262 accel_dev->hw_device = hw_data; 263 switch (ent->device) { 264 case ADF_DH895XCC_PCI_DEVICE_ID: 265 adf_init_hw_data_dh895xcc(accel_dev->hw_device); 266 break; 267 default: 268 return -ENODEV; 269 } 270 accel_pci_dev = &accel_dev->accel_pci_dev; 271 pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); 272 pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET, 273 &hw_data->fuses); 274 275 /* Get Accelerators and Accelerators Engines masks */ 276 hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); 277 hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); 278 accel_pci_dev->sku = hw_data->get_sku(hw_data); 279 accel_pci_dev->pci_dev = pdev; 280 /* If the device has no acceleration engines then ignore it. */ 281 if (!hw_data->accel_mask || !hw_data->ae_mask || 282 ((~hw_data->ae_mask) & 0x01)) { 283 dev_err(&pdev->dev, "No acceleration units found"); 284 ret = -EFAULT; 285 goto out_err; 286 } 287 288 /* Create dev top level debugfs entry */ 289 snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX, 290 hw_data->dev_class->name, hw_data->instance_id); 291 accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); 292 if (!accel_dev->debugfs_dir) { 293 dev_err(&pdev->dev, "Could not create debugfs dir\n"); 294 ret = -EINVAL; 295 goto out_err; 296 } 297 298 /* Create device configuration table */ 299 ret = adf_cfg_dev_add(accel_dev); 300 if (ret) 301 goto out_err; 302 303 /* enable PCI device */ 304 if (pci_enable_device(pdev)) { 305 ret = -EFAULT; 306 goto out_err; 307 } 308 309 /* set dma identifier */ 310 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 311 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { 312 dev_err(&pdev->dev, "No usable DMA configuration\n"); 313 ret = -EFAULT; 314 goto out_err; 315 } else { 316 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 317 } 318 319 } else { 320 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 321 } 322 323 if (pci_request_regions(pdev, adf_driver_name)) { 324 ret = -EFAULT; 325 goto out_err; 326 } 327 328 /* Read accelerator capabilities mask */ 329 pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET, 330 &hw_data->accel_capabilities_mask); 331 332 /* Find and map all the device's BARS */ 333 for (i = 0; i < ADF_PCI_MAX_BARS; i++) { 334 struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; 335 336 bar_nr = i * 2; 337 bar->base_addr = pci_resource_start(pdev, bar_nr); 338 if (!bar->base_addr) 339 break; 340 bar->size = pci_resource_len(pdev, bar_nr); 341 bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); 342 if (!bar->virt_addr) { 343 dev_err(&pdev->dev, "Failed to map BAR %d\n", i); 344 ret = -EFAULT; 345 goto out_err; 346 } 347 } 348 pci_set_master(pdev); 349 350 if (adf_enable_aer(accel_dev, &adf_driver)) { 351 dev_err(&pdev->dev, "Failed to enable aer\n"); 352 ret = -EFAULT; 353 goto out_err; 354 } 355 356 if (pci_save_state(pdev)) { 357 dev_err(&pdev->dev, "Failed to save pci state\n"); 358 ret = -ENOMEM; 359 goto out_err; 360 } 361 362 ret = adf_dev_configure(accel_dev); 363 if (ret) 364 goto out_err; 365 366 ret = adf_dev_init(accel_dev); 367 if (ret) 368 goto out_err; 369 370 ret = adf_dev_start(accel_dev); 371 if (ret) { 372 adf_dev_stop(accel_dev); 373 goto out_err; 374 } 375 376 return 0; 377out_err: 378 adf_cleanup_accel(accel_dev); 379 return ret; 380} 381 382static void adf_remove(struct pci_dev *pdev) 383{ 384 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); 385 386 if (!accel_dev) { 387 pr_err("QAT: Driver removal failed\n"); 388 return; 389 } 390 if (adf_dev_stop(accel_dev)) 391 dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); 392 adf_disable_aer(accel_dev); 393 adf_cleanup_accel(accel_dev); 394} 395 396static int __init adfdrv_init(void) 397{ 398 request_module("intel_qat"); 399 if (qat_admin_register()) 400 return -EFAULT; 401 402 if (pci_register_driver(&adf_driver)) { 403 pr_err("QAT: Driver initialization failed\n"); 404 return -EFAULT; 405 } 406 return 0; 407} 408 409static void __exit adfdrv_release(void) 410{ 411 pci_unregister_driver(&adf_driver); 412 qat_admin_unregister(); 413} 414 415module_init(adfdrv_init); 416module_exit(adfdrv_release); 417 418MODULE_LICENSE("Dual BSD/GPL"); 419MODULE_AUTHOR("Intel"); 420MODULE_FIRMWARE("qat_895xcc.bin"); 421MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); 422