1/* 2 This file is provided under a dual BSD/GPLv2 license. When using or 3 redistributing this file, you may do so under either license. 4 5 GPL LICENSE SUMMARY 6 Copyright(c) 2014 Intel Corporation. 7 This program is free software; you can redistribute it and/or modify 8 it under the terms of version 2 of the GNU General Public License as 9 published by the Free Software Foundation. 10 11 This program is distributed in the hope that it will be useful, but 12 WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 General Public License for more details. 15 16 Contact Information: 17 qat-linux@intel.com 18 19 BSD LICENSE 20 Copyright(c) 2014 Intel Corporation. 21 Redistribution and use in source and binary forms, with or without 22 modification, are permitted provided that the following conditions 23 are met: 24 25 * Redistributions of source code must retain the above copyright 26 notice, this list of conditions and the following disclaimer. 27 * Redistributions in binary form must reproduce the above copyright 28 notice, this list of conditions and the following disclaimer in 29 the documentation and/or other materials provided with the 30 distribution. 31 * Neither the name of Intel Corporation nor the names of its 32 contributors may be used to endorse or promote products derived 33 from this software without specific prior written permission. 34 35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46*/ 47#include <linux/mutex.h> 48#include <linux/list.h> 49#include <linux/bitops.h> 50#include <linux/delay.h> 51#include "adf_accel_devices.h" 52#include "adf_cfg.h" 53#include "adf_common_drv.h" 54 55static LIST_HEAD(service_table); 56static DEFINE_MUTEX(service_lock); 57 58static void adf_service_add(struct service_hndl *service) 59{ 60 mutex_lock(&service_lock); 61 list_add(&service->list, &service_table); 62 mutex_unlock(&service_lock); 63} 64 65/** 66 * adf_service_register() - Register acceleration service in the accel framework 67 * @service: Pointer to the service 68 * 69 * Function adds the acceleration service to the acceleration framework. 70 * To be used by QAT device specific drivers. 71 * 72 * Return: 0 on success, error code otherwise. 73 */ 74int adf_service_register(struct service_hndl *service) 75{ 76 service->init_status = 0; 77 service->start_status = 0; 78 adf_service_add(service); 79 return 0; 80} 81EXPORT_SYMBOL_GPL(adf_service_register); 82 83static void adf_service_remove(struct service_hndl *service) 84{ 85 mutex_lock(&service_lock); 86 list_del(&service->list); 87 mutex_unlock(&service_lock); 88} 89 90/** 91 * adf_service_unregister() - Unregister acceleration service from the framework 92 * @service: Pointer to the service 93 * 94 * Function remove the acceleration service from the acceleration framework. 95 * To be used by QAT device specific drivers. 96 * 97 * Return: 0 on success, error code otherwise. 98 */ 99int adf_service_unregister(struct service_hndl *service) 100{ 101 if (service->init_status || service->start_status) { 102 pr_err("QAT: Could not remove active service\n"); 103 return -EFAULT; 104 } 105 adf_service_remove(service); 106 return 0; 107} 108EXPORT_SYMBOL_GPL(adf_service_unregister); 109 110/** 111 * adf_dev_init() - Init data structures and services for the given accel device 112 * @accel_dev: Pointer to acceleration device. 113 * 114 * Initialize the ring data structures and the admin comms and arbitration 115 * services. 116 * 117 * Return: 0 on success, error code otherwise. 118 */ 119int adf_dev_init(struct adf_accel_dev *accel_dev) 120{ 121 struct service_hndl *service; 122 struct list_head *list_itr; 123 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 124 125 if (!hw_data) { 126 dev_err(&GET_DEV(accel_dev), 127 "Failed to init device - hw_data not set\n"); 128 return -EFAULT; 129 } 130 131 if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) { 132 dev_err(&GET_DEV(accel_dev), "Device not configured\n"); 133 return -EFAULT; 134 } 135 136 if (adf_init_etr_data(accel_dev)) { 137 dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n"); 138 return -EFAULT; 139 } 140 141 if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) { 142 dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n"); 143 return -EFAULT; 144 } 145 146 if (hw_data->init_arb && hw_data->init_arb(accel_dev)) { 147 dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n"); 148 return -EFAULT; 149 } 150 151 hw_data->enable_ints(accel_dev); 152 153 if (adf_ae_init(accel_dev)) { 154 dev_err(&GET_DEV(accel_dev), 155 "Failed to initialise Acceleration Engine\n"); 156 return -EFAULT; 157 } 158 set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status); 159 160 if (adf_ae_fw_load(accel_dev)) { 161 dev_err(&GET_DEV(accel_dev), 162 "Failed to load acceleration FW\n"); 163 return -EFAULT; 164 } 165 set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status); 166 167 if (hw_data->alloc_irq(accel_dev)) { 168 dev_err(&GET_DEV(accel_dev), "Failed to allocate interrupts\n"); 169 return -EFAULT; 170 } 171 set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status); 172 173 /* 174 * Subservice initialisation is divided into two stages: init and start. 175 * This is to facilitate any ordering dependencies between services 176 * prior to starting any of the accelerators. 177 */ 178 list_for_each(list_itr, &service_table) { 179 service = list_entry(list_itr, struct service_hndl, list); 180 if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { 181 dev_err(&GET_DEV(accel_dev), 182 "Failed to initialise service %s\n", 183 service->name); 184 return -EFAULT; 185 } 186 set_bit(accel_dev->accel_id, &service->init_status); 187 } 188 189 hw_data->enable_error_correction(accel_dev); 190 hw_data->enable_vf2pf_comms(accel_dev); 191 192 return 0; 193} 194EXPORT_SYMBOL_GPL(adf_dev_init); 195 196/** 197 * adf_dev_start() - Start acceleration service for the given accel device 198 * @accel_dev: Pointer to acceleration device. 199 * 200 * Function notifies all the registered services that the acceleration device 201 * is ready to be used. 202 * To be used by QAT device specific drivers. 203 * 204 * Return: 0 on success, error code otherwise. 205 */ 206int adf_dev_start(struct adf_accel_dev *accel_dev) 207{ 208 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 209 struct service_hndl *service; 210 struct list_head *list_itr; 211 212 set_bit(ADF_STATUS_STARTING, &accel_dev->status); 213 214 if (adf_ae_start(accel_dev)) { 215 dev_err(&GET_DEV(accel_dev), "AE Start Failed\n"); 216 return -EFAULT; 217 } 218 set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); 219 220 if (hw_data->send_admin_init(accel_dev)) { 221 dev_err(&GET_DEV(accel_dev), "Failed to send init message\n"); 222 return -EFAULT; 223 } 224 225 list_for_each(list_itr, &service_table) { 226 service = list_entry(list_itr, struct service_hndl, list); 227 if (service->event_hld(accel_dev, ADF_EVENT_START)) { 228 dev_err(&GET_DEV(accel_dev), 229 "Failed to start service %s\n", 230 service->name); 231 return -EFAULT; 232 } 233 set_bit(accel_dev->accel_id, &service->start_status); 234 } 235 236 clear_bit(ADF_STATUS_STARTING, &accel_dev->status); 237 set_bit(ADF_STATUS_STARTED, &accel_dev->status); 238 239 if (!list_empty(&accel_dev->crypto_list) && 240 (qat_algs_register() || qat_asym_algs_register())) { 241 dev_err(&GET_DEV(accel_dev), 242 "Failed to register crypto algs\n"); 243 set_bit(ADF_STATUS_STARTING, &accel_dev->status); 244 clear_bit(ADF_STATUS_STARTED, &accel_dev->status); 245 return -EFAULT; 246 } 247 return 0; 248} 249EXPORT_SYMBOL_GPL(adf_dev_start); 250 251/** 252 * adf_dev_stop() - Stop acceleration service for the given accel device 253 * @accel_dev: Pointer to acceleration device. 254 * 255 * Function notifies all the registered services that the acceleration device 256 * is shuting down. 257 * To be used by QAT device specific drivers. 258 * 259 * Return: 0 on success, error code otherwise. 260 */ 261int adf_dev_stop(struct adf_accel_dev *accel_dev) 262{ 263 struct service_hndl *service; 264 struct list_head *list_itr; 265 bool wait = false; 266 int ret; 267 268 if (!adf_dev_started(accel_dev) && 269 !test_bit(ADF_STATUS_STARTING, &accel_dev->status)) { 270 return 0; 271 } 272 clear_bit(ADF_STATUS_STARTING, &accel_dev->status); 273 clear_bit(ADF_STATUS_STARTED, &accel_dev->status); 274 275 if (!list_empty(&accel_dev->crypto_list)) { 276 qat_algs_unregister(); 277 qat_asym_algs_unregister(); 278 } 279 280 list_for_each(list_itr, &service_table) { 281 service = list_entry(list_itr, struct service_hndl, list); 282 if (!test_bit(accel_dev->accel_id, &service->start_status)) 283 continue; 284 ret = service->event_hld(accel_dev, ADF_EVENT_STOP); 285 if (!ret) { 286 clear_bit(accel_dev->accel_id, &service->start_status); 287 } else if (ret == -EAGAIN) { 288 wait = true; 289 clear_bit(accel_dev->accel_id, &service->start_status); 290 } 291 } 292 293 if (wait) 294 msleep(100); 295 296 if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) { 297 if (adf_ae_stop(accel_dev)) 298 dev_err(&GET_DEV(accel_dev), "failed to stop AE\n"); 299 else 300 clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); 301 } 302 303 return 0; 304} 305EXPORT_SYMBOL_GPL(adf_dev_stop); 306 307/** 308 * adf_dev_shutdown() - shutdown acceleration services and data strucutures 309 * @accel_dev: Pointer to acceleration device 310 * 311 * Cleanup the ring data structures and the admin comms and arbitration 312 * services. 313 */ 314void adf_dev_shutdown(struct adf_accel_dev *accel_dev) 315{ 316 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 317 struct service_hndl *service; 318 struct list_head *list_itr; 319 320 if (!hw_data) { 321 dev_err(&GET_DEV(accel_dev), 322 "QAT: Failed to shutdown device - hw_data not set\n"); 323 return; 324 } 325 326 if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) { 327 adf_ae_fw_release(accel_dev); 328 clear_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status); 329 } 330 331 if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) { 332 if (adf_ae_shutdown(accel_dev)) 333 dev_err(&GET_DEV(accel_dev), 334 "Failed to shutdown Accel Engine\n"); 335 else 336 clear_bit(ADF_STATUS_AE_INITIALISED, 337 &accel_dev->status); 338 } 339 340 list_for_each(list_itr, &service_table) { 341 service = list_entry(list_itr, struct service_hndl, list); 342 if (!test_bit(accel_dev->accel_id, &service->init_status)) 343 continue; 344 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) 345 dev_err(&GET_DEV(accel_dev), 346 "Failed to shutdown service %s\n", 347 service->name); 348 else 349 clear_bit(accel_dev->accel_id, &service->init_status); 350 } 351 352 if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) { 353 hw_data->free_irq(accel_dev); 354 clear_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status); 355 } 356 357 /* Delete configuration only if not restarting */ 358 if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) 359 adf_cfg_del_all(accel_dev); 360 361 if (hw_data->exit_arb) 362 hw_data->exit_arb(accel_dev); 363 364 if (hw_data->exit_admin_comms) 365 hw_data->exit_admin_comms(accel_dev); 366 367 hw_data->disable_iov(accel_dev); 368 adf_cleanup_etr_data(accel_dev); 369} 370EXPORT_SYMBOL_GPL(adf_dev_shutdown); 371 372int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev) 373{ 374 struct service_hndl *service; 375 struct list_head *list_itr; 376 377 list_for_each(list_itr, &service_table) { 378 service = list_entry(list_itr, struct service_hndl, list); 379 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) 380 dev_err(&GET_DEV(accel_dev), 381 "Failed to restart service %s.\n", 382 service->name); 383 } 384 return 0; 385} 386 387int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev) 388{ 389 struct service_hndl *service; 390 struct list_head *list_itr; 391 392 list_for_each(list_itr, &service_table) { 393 service = list_entry(list_itr, struct service_hndl, list); 394 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) 395 dev_err(&GET_DEV(accel_dev), 396 "Failed to restart service %s.\n", 397 service->name); 398 } 399 return 0; 400} 401