1 /*
2 This file is provided under a dual BSD/GPLv2 license. When using or
3 redistributing this file, you may do so under either license.
4
5 GPL LICENSE SUMMARY
6 Copyright(c) 2014 Intel Corporation.
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of version 2 of the GNU General Public License as
9 published by the Free Software Foundation.
10
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
15
16 Contact Information:
17 qat-linux@intel.com
18
19 BSD LICENSE
20 Copyright(c) 2014 Intel Corporation.
21 Redistribution and use in source and binary forms, with or without
22 modification, are permitted provided that the following conditions
23 are met:
24
25 * Redistributions of source code must retain the above copyright
26 notice, this list of conditions and the following disclaimer.
27 * Redistributions in binary form must reproduce the above copyright
28 notice, this list of conditions and the following disclaimer in
29 the documentation and/or other materials provided with the
30 distribution.
31 * Neither the name of Intel Corporation nor the names of its
32 contributors may be used to endorse or promote products derived
33 from this software without specific prior written permission.
34
35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 */
47 #include <linux/mutex.h>
48 #include <linux/list.h>
49 #include <linux/bitops.h>
50 #include <linux/delay.h>
51 #include "adf_accel_devices.h"
52 #include "adf_cfg.h"
53 #include "adf_common_drv.h"
54
55 static LIST_HEAD(service_table);
56 static DEFINE_MUTEX(service_lock);
57
adf_service_add(struct service_hndl * service)58 static void adf_service_add(struct service_hndl *service)
59 {
60 mutex_lock(&service_lock);
61 list_add(&service->list, &service_table);
62 mutex_unlock(&service_lock);
63 }
64
65 /**
66 * adf_service_register() - Register acceleration service in the accel framework
67 * @service: Pointer to the service
68 *
69 * Function adds the acceleration service to the acceleration framework.
70 * To be used by QAT device specific drivers.
71 *
72 * Return: 0 on success, error code othewise.
73 */
adf_service_register(struct service_hndl * service)74 int adf_service_register(struct service_hndl *service)
75 {
76 service->init_status = 0;
77 service->start_status = 0;
78 adf_service_add(service);
79 return 0;
80 }
81 EXPORT_SYMBOL_GPL(adf_service_register);
82
adf_service_remove(struct service_hndl * service)83 static void adf_service_remove(struct service_hndl *service)
84 {
85 mutex_lock(&service_lock);
86 list_del(&service->list);
87 mutex_unlock(&service_lock);
88 }
89
90 /**
91 * adf_service_unregister() - Unregister acceleration service from the framework
92 * @service: Pointer to the service
93 *
94 * Function remove the acceleration service from the acceleration framework.
95 * To be used by QAT device specific drivers.
96 *
97 * Return: 0 on success, error code othewise.
98 */
adf_service_unregister(struct service_hndl * service)99 int adf_service_unregister(struct service_hndl *service)
100 {
101 if (service->init_status || service->start_status) {
102 pr_err("QAT: Could not remove active service\n");
103 return -EFAULT;
104 }
105 adf_service_remove(service);
106 return 0;
107 }
108 EXPORT_SYMBOL_GPL(adf_service_unregister);
109
110 /**
111 * adf_dev_init() - Init data structures and services for the given accel device
112 * @accel_dev: Pointer to acceleration device.
113 *
114 * Initialize the ring data structures and the admin comms and arbitration
115 * services.
116 *
117 * Return: 0 on success, error code othewise.
118 */
adf_dev_init(struct adf_accel_dev * accel_dev)119 int adf_dev_init(struct adf_accel_dev *accel_dev)
120 {
121 struct service_hndl *service;
122 struct list_head *list_itr;
123 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
124
125 if (!hw_data) {
126 dev_err(&GET_DEV(accel_dev),
127 "Failed to init device - hw_data not set\n");
128 return -EFAULT;
129 }
130
131 if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) {
132 dev_err(&GET_DEV(accel_dev), "Device not configured\n");
133 return -EFAULT;
134 }
135
136 if (adf_init_etr_data(accel_dev)) {
137 dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n");
138 return -EFAULT;
139 }
140
141 if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) {
142 dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n");
143 return -EFAULT;
144 }
145
146 if (hw_data->init_arb && hw_data->init_arb(accel_dev)) {
147 dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n");
148 return -EFAULT;
149 }
150
151 hw_data->enable_ints(accel_dev);
152
153 if (adf_ae_init(accel_dev)) {
154 dev_err(&GET_DEV(accel_dev),
155 "Failed to initialise Acceleration Engine\n");
156 return -EFAULT;
157 }
158 set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status);
159
160 if (adf_ae_fw_load(accel_dev)) {
161 dev_err(&GET_DEV(accel_dev),
162 "Failed to load acceleration FW\n");
163 return -EFAULT;
164 }
165 set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
166
167 if (hw_data->alloc_irq(accel_dev)) {
168 dev_err(&GET_DEV(accel_dev), "Failed to allocate interrupts\n");
169 return -EFAULT;
170 }
171 set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
172
173 /*
174 * Subservice initialisation is divided into two stages: init and start.
175 * This is to facilitate any ordering dependencies between services
176 * prior to starting any of the accelerators.
177 */
178 list_for_each(list_itr, &service_table) {
179 service = list_entry(list_itr, struct service_hndl, list);
180 if (!service->admin)
181 continue;
182 if (service->event_hld(accel_dev, ADF_EVENT_INIT)) {
183 dev_err(&GET_DEV(accel_dev),
184 "Failed to initialise service %s\n",
185 service->name);
186 return -EFAULT;
187 }
188 set_bit(accel_dev->accel_id, &service->init_status);
189 }
190 list_for_each(list_itr, &service_table) {
191 service = list_entry(list_itr, struct service_hndl, list);
192 if (service->admin)
193 continue;
194 if (service->event_hld(accel_dev, ADF_EVENT_INIT)) {
195 dev_err(&GET_DEV(accel_dev),
196 "Failed to initialise service %s\n",
197 service->name);
198 return -EFAULT;
199 }
200 set_bit(accel_dev->accel_id, &service->init_status);
201 }
202
203 hw_data->enable_error_correction(accel_dev);
204
205 return 0;
206 }
207 EXPORT_SYMBOL_GPL(adf_dev_init);
208
209 /**
210 * adf_dev_start() - Start acceleration service for the given accel device
211 * @accel_dev: Pointer to acceleration device.
212 *
213 * Function notifies all the registered services that the acceleration device
214 * is ready to be used.
215 * To be used by QAT device specific drivers.
216 *
217 * Return: 0 on success, error code othewise.
218 */
adf_dev_start(struct adf_accel_dev * accel_dev)219 int adf_dev_start(struct adf_accel_dev *accel_dev)
220 {
221 struct service_hndl *service;
222 struct list_head *list_itr;
223
224 set_bit(ADF_STATUS_STARTING, &accel_dev->status);
225
226 if (adf_ae_start(accel_dev)) {
227 dev_err(&GET_DEV(accel_dev), "AE Start Failed\n");
228 return -EFAULT;
229 }
230 set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
231
232 list_for_each(list_itr, &service_table) {
233 service = list_entry(list_itr, struct service_hndl, list);
234 if (!service->admin)
235 continue;
236 if (service->event_hld(accel_dev, ADF_EVENT_START)) {
237 dev_err(&GET_DEV(accel_dev),
238 "Failed to start service %s\n",
239 service->name);
240 return -EFAULT;
241 }
242 set_bit(accel_dev->accel_id, &service->start_status);
243 }
244 list_for_each(list_itr, &service_table) {
245 service = list_entry(list_itr, struct service_hndl, list);
246 if (service->admin)
247 continue;
248 if (service->event_hld(accel_dev, ADF_EVENT_START)) {
249 dev_err(&GET_DEV(accel_dev),
250 "Failed to start service %s\n",
251 service->name);
252 return -EFAULT;
253 }
254 set_bit(accel_dev->accel_id, &service->start_status);
255 }
256
257 clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
258 set_bit(ADF_STATUS_STARTED, &accel_dev->status);
259
260 if (qat_algs_register()) {
261 dev_err(&GET_DEV(accel_dev),
262 "Failed to register crypto algs\n");
263 set_bit(ADF_STATUS_STARTING, &accel_dev->status);
264 clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
265 return -EFAULT;
266 }
267 return 0;
268 }
269 EXPORT_SYMBOL_GPL(adf_dev_start);
270
271 /**
272 * adf_dev_stop() - Stop acceleration service for the given accel device
273 * @accel_dev: Pointer to acceleration device.
274 *
275 * Function notifies all the registered services that the acceleration device
276 * is shuting down.
277 * To be used by QAT device specific drivers.
278 *
279 * Return: 0 on success, error code othewise.
280 */
adf_dev_stop(struct adf_accel_dev * accel_dev)281 int adf_dev_stop(struct adf_accel_dev *accel_dev)
282 {
283 struct service_hndl *service;
284 struct list_head *list_itr;
285 bool wait = false;
286 int ret;
287
288 if (!adf_dev_started(accel_dev) &&
289 !test_bit(ADF_STATUS_STARTING, &accel_dev->status)) {
290 return 0;
291 }
292 clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
293 clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
294
295 if (qat_algs_unregister())
296 dev_err(&GET_DEV(accel_dev),
297 "Failed to unregister crypto algs\n");
298
299 list_for_each(list_itr, &service_table) {
300 service = list_entry(list_itr, struct service_hndl, list);
301 if (service->admin)
302 continue;
303 if (!test_bit(accel_dev->accel_id, &service->start_status))
304 continue;
305 ret = service->event_hld(accel_dev, ADF_EVENT_STOP);
306 if (!ret) {
307 clear_bit(accel_dev->accel_id, &service->start_status);
308 } else if (ret == -EAGAIN) {
309 wait = true;
310 clear_bit(accel_dev->accel_id, &service->start_status);
311 }
312 }
313 list_for_each(list_itr, &service_table) {
314 service = list_entry(list_itr, struct service_hndl, list);
315 if (!service->admin)
316 continue;
317 if (!test_bit(accel_dev->accel_id, &service->start_status))
318 continue;
319 if (service->event_hld(accel_dev, ADF_EVENT_STOP))
320 dev_err(&GET_DEV(accel_dev),
321 "Failed to shutdown service %s\n",
322 service->name);
323 else
324 clear_bit(accel_dev->accel_id, &service->start_status);
325 }
326
327 if (wait)
328 msleep(100);
329
330 if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) {
331 if (adf_ae_stop(accel_dev))
332 dev_err(&GET_DEV(accel_dev), "failed to stop AE\n");
333 else
334 clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
335 }
336
337 return 0;
338 }
339 EXPORT_SYMBOL_GPL(adf_dev_stop);
340
341 /**
342 * adf_dev_shutdown() - shutdown acceleration services and data strucutures
343 * @accel_dev: Pointer to acceleration device
344 *
345 * Cleanup the ring data structures and the admin comms and arbitration
346 * services.
347 */
adf_dev_shutdown(struct adf_accel_dev * accel_dev)348 void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
349 {
350 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
351 struct service_hndl *service;
352 struct list_head *list_itr;
353
354 if (!hw_data) {
355 dev_err(&GET_DEV(accel_dev),
356 "QAT: Failed to shutdown device - hw_data not set\n");
357 return;
358 }
359
360 if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) {
361 adf_ae_fw_release(accel_dev);
362 clear_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
363 }
364
365 if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) {
366 if (adf_ae_shutdown(accel_dev))
367 dev_err(&GET_DEV(accel_dev),
368 "Failed to shutdown Accel Engine\n");
369 else
370 clear_bit(ADF_STATUS_AE_INITIALISED,
371 &accel_dev->status);
372 }
373
374 list_for_each(list_itr, &service_table) {
375 service = list_entry(list_itr, struct service_hndl, list);
376 if (service->admin)
377 continue;
378 if (!test_bit(accel_dev->accel_id, &service->init_status))
379 continue;
380 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN))
381 dev_err(&GET_DEV(accel_dev),
382 "Failed to shutdown service %s\n",
383 service->name);
384 else
385 clear_bit(accel_dev->accel_id, &service->init_status);
386 }
387 list_for_each(list_itr, &service_table) {
388 service = list_entry(list_itr, struct service_hndl, list);
389 if (!service->admin)
390 continue;
391 if (!test_bit(accel_dev->accel_id, &service->init_status))
392 continue;
393 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN))
394 dev_err(&GET_DEV(accel_dev),
395 "Failed to shutdown service %s\n",
396 service->name);
397 else
398 clear_bit(accel_dev->accel_id, &service->init_status);
399 }
400
401 if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) {
402 hw_data->free_irq(accel_dev);
403 clear_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
404 }
405
406 /* Delete configuration only if not restarting */
407 if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
408 adf_cfg_del_all(accel_dev);
409
410 if (hw_data->exit_arb)
411 hw_data->exit_arb(accel_dev);
412
413 if (hw_data->exit_admin_comms)
414 hw_data->exit_admin_comms(accel_dev);
415
416 adf_cleanup_etr_data(accel_dev);
417 }
418 EXPORT_SYMBOL_GPL(adf_dev_shutdown);
419
adf_dev_restarting_notify(struct adf_accel_dev * accel_dev)420 int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev)
421 {
422 struct service_hndl *service;
423 struct list_head *list_itr;
424
425 list_for_each(list_itr, &service_table) {
426 service = list_entry(list_itr, struct service_hndl, list);
427 if (service->admin)
428 continue;
429 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING))
430 dev_err(&GET_DEV(accel_dev),
431 "Failed to restart service %s.\n",
432 service->name);
433 }
434 list_for_each(list_itr, &service_table) {
435 service = list_entry(list_itr, struct service_hndl, list);
436 if (!service->admin)
437 continue;
438 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING))
439 dev_err(&GET_DEV(accel_dev),
440 "Failed to restart service %s.\n",
441 service->name);
442 }
443 return 0;
444 }
445
adf_dev_restarted_notify(struct adf_accel_dev * accel_dev)446 int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev)
447 {
448 struct service_hndl *service;
449 struct list_head *list_itr;
450
451 list_for_each(list_itr, &service_table) {
452 service = list_entry(list_itr, struct service_hndl, list);
453 if (service->admin)
454 continue;
455 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED))
456 dev_err(&GET_DEV(accel_dev),
457 "Failed to restart service %s.\n",
458 service->name);
459 }
460 list_for_each(list_itr, &service_table) {
461 service = list_entry(list_itr, struct service_hndl, list);
462 if (!service->admin)
463 continue;
464 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED))
465 dev_err(&GET_DEV(accel_dev),
466 "Failed to restart service %s.\n",
467 service->name);
468 }
469 return 0;
470 }
471