1/*
2 *  IBM eServer eHCA Infiniband device driver for Linux on POWER
3 *
4 *  module start stop, hca detection
5 *
6 *  Authors: Heiko J Schick <schickhj@de.ibm.com>
7 *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
8 *           Joachim Fenkes <fenkes@de.ibm.com>
9 *
10 *  Copyright (c) 2005 IBM Corporation
11 *
12 *  All rights reserved.
13 *
14 *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
15 *  BSD.
16 *
17 * OpenIB BSD License
18 *
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions are met:
21 *
22 * Redistributions of source code must retain the above copyright notice, this
23 * list of conditions and the following disclaimer.
24 *
25 * Redistributions in binary form must reproduce the above copyright notice,
26 * this list of conditions and the following disclaimer in the documentation
27 * and/or other materials
28 * provided with the distribution.
29 *
30 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
31 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
32 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
34 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
37 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
38 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
40 * POSSIBILITY OF SUCH DAMAGE.
41 */
42
43#ifdef CONFIG_PPC_64K_PAGES
44#include <linux/slab.h>
45#endif
46
47#include <linux/notifier.h>
48#include <linux/memory.h>
49#include "ehca_classes.h"
50#include "ehca_iverbs.h"
51#include "ehca_mrmw.h"
52#include "ehca_tools.h"
53#include "hcp_if.h"
54
55#define HCAD_VERSION "0029"
56
57MODULE_LICENSE("Dual BSD/GPL");
58MODULE_AUTHOR("Christoph Raisch <raisch@de.ibm.com>");
59MODULE_DESCRIPTION("IBM eServer HCA InfiniBand Device Driver");
60MODULE_VERSION(HCAD_VERSION);
61
62static bool ehca_open_aqp1    = 0;
63static int ehca_hw_level      = 0;
64static bool ehca_poll_all_eqs = 1;
65
66int ehca_debug_level   = 0;
67int ehca_nr_ports      = -1;
68bool ehca_use_hp_mr    = 0;
69int ehca_port_act_time = 30;
70int ehca_static_rate   = -1;
71bool ehca_scaling_code = 0;
72int ehca_lock_hcalls   = -1;
73int ehca_max_cq        = -1;
74int ehca_max_qp        = -1;
75
76module_param_named(open_aqp1,     ehca_open_aqp1,     bool, S_IRUGO);
77module_param_named(debug_level,   ehca_debug_level,   int,  S_IRUGO);
78module_param_named(hw_level,      ehca_hw_level,      int,  S_IRUGO);
79module_param_named(nr_ports,      ehca_nr_ports,      int,  S_IRUGO);
80module_param_named(use_hp_mr,     ehca_use_hp_mr,     bool, S_IRUGO);
81module_param_named(port_act_time, ehca_port_act_time, int,  S_IRUGO);
82module_param_named(poll_all_eqs,  ehca_poll_all_eqs,  bool, S_IRUGO);
83module_param_named(static_rate,   ehca_static_rate,   int,  S_IRUGO);
84module_param_named(scaling_code,  ehca_scaling_code,  bool, S_IRUGO);
85module_param_named(lock_hcalls,   ehca_lock_hcalls,   bint, S_IRUGO);
86module_param_named(number_of_cqs, ehca_max_cq,        int,  S_IRUGO);
87module_param_named(number_of_qps, ehca_max_qp,        int,  S_IRUGO);
88
89MODULE_PARM_DESC(open_aqp1,
90		 "Open AQP1 on startup (default: no)");
91MODULE_PARM_DESC(debug_level,
92		 "Amount of debug output (0: none (default), 1: traces, "
93		 "2: some dumps, 3: lots)");
94MODULE_PARM_DESC(hw_level,
95		 "Hardware level (0: autosensing (default), "
96		 "0x10..0x14: eHCA, 0x20..0x23: eHCA2)");
97MODULE_PARM_DESC(nr_ports,
98		 "number of connected ports (-1: autodetect (default), "
99		 "1: port one only, 2: two ports)");
100MODULE_PARM_DESC(use_hp_mr,
101		 "Use high performance MRs (default: no)");
102MODULE_PARM_DESC(port_act_time,
103		 "Time to wait for port activation (default: 30 sec)");
104MODULE_PARM_DESC(poll_all_eqs,
105		 "Poll all event queues periodically (default: yes)");
106MODULE_PARM_DESC(static_rate,
107		 "Set permanent static rate (default: no static rate)");
108MODULE_PARM_DESC(scaling_code,
109		 "Enable scaling code (default: no)");
110MODULE_PARM_DESC(lock_hcalls,
111		 "Serialize all hCalls made by the driver "
112		 "(default: autodetect)");
113MODULE_PARM_DESC(number_of_cqs,
114		"Max number of CQs which can be allocated "
115		"(default: autodetect)");
116MODULE_PARM_DESC(number_of_qps,
117		"Max number of QPs which can be allocated "
118		"(default: autodetect)");
119
120DEFINE_RWLOCK(ehca_qp_idr_lock);
121DEFINE_RWLOCK(ehca_cq_idr_lock);
122DEFINE_IDR(ehca_qp_idr);
123DEFINE_IDR(ehca_cq_idr);
124
125static LIST_HEAD(shca_list); /* list of all registered ehcas */
126DEFINE_SPINLOCK(shca_list_lock);
127
128static struct timer_list poll_eqs_timer;
129
130#ifdef CONFIG_PPC_64K_PAGES
131static struct kmem_cache *ctblk_cache;
132
133void *ehca_alloc_fw_ctrlblock(gfp_t flags)
134{
135	void *ret = kmem_cache_zalloc(ctblk_cache, flags);
136	if (!ret)
137		ehca_gen_err("Out of memory for ctblk");
138	return ret;
139}
140
141void ehca_free_fw_ctrlblock(void *ptr)
142{
143	if (ptr)
144		kmem_cache_free(ctblk_cache, ptr);
145
146}
147#endif
148
149int ehca2ib_return_code(u64 ehca_rc)
150{
151	switch (ehca_rc) {
152	case H_SUCCESS:
153		return 0;
154	case H_RESOURCE:             /* Resource in use */
155	case H_BUSY:
156		return -EBUSY;
157	case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
158	case H_CONSTRAINED:          /* resource constraint */
159	case H_NO_MEM:
160		return -ENOMEM;
161	default:
162		return -EINVAL;
163	}
164}
165
166static int ehca_create_slab_caches(void)
167{
168	int ret;
169
170	ret = ehca_init_pd_cache();
171	if (ret) {
172		ehca_gen_err("Cannot create PD SLAB cache.");
173		return ret;
174	}
175
176	ret = ehca_init_cq_cache();
177	if (ret) {
178		ehca_gen_err("Cannot create CQ SLAB cache.");
179		goto create_slab_caches2;
180	}
181
182	ret = ehca_init_qp_cache();
183	if (ret) {
184		ehca_gen_err("Cannot create QP SLAB cache.");
185		goto create_slab_caches3;
186	}
187
188	ret = ehca_init_av_cache();
189	if (ret) {
190		ehca_gen_err("Cannot create AV SLAB cache.");
191		goto create_slab_caches4;
192	}
193
194	ret = ehca_init_mrmw_cache();
195	if (ret) {
196		ehca_gen_err("Cannot create MR&MW SLAB cache.");
197		goto create_slab_caches5;
198	}
199
200	ret = ehca_init_small_qp_cache();
201	if (ret) {
202		ehca_gen_err("Cannot create small queue SLAB cache.");
203		goto create_slab_caches6;
204	}
205
206#ifdef CONFIG_PPC_64K_PAGES
207	ctblk_cache = kmem_cache_create("ehca_cache_ctblk",
208					EHCA_PAGESIZE, H_CB_ALIGNMENT,
209					SLAB_HWCACHE_ALIGN,
210					NULL);
211	if (!ctblk_cache) {
212		ehca_gen_err("Cannot create ctblk SLAB cache.");
213		ehca_cleanup_small_qp_cache();
214		ret = -ENOMEM;
215		goto create_slab_caches6;
216	}
217#endif
218	return 0;
219
220create_slab_caches6:
221	ehca_cleanup_mrmw_cache();
222
223create_slab_caches5:
224	ehca_cleanup_av_cache();
225
226create_slab_caches4:
227	ehca_cleanup_qp_cache();
228
229create_slab_caches3:
230	ehca_cleanup_cq_cache();
231
232create_slab_caches2:
233	ehca_cleanup_pd_cache();
234
235	return ret;
236}
237
238static void ehca_destroy_slab_caches(void)
239{
240	ehca_cleanup_small_qp_cache();
241	ehca_cleanup_mrmw_cache();
242	ehca_cleanup_av_cache();
243	ehca_cleanup_qp_cache();
244	ehca_cleanup_cq_cache();
245	ehca_cleanup_pd_cache();
246#ifdef CONFIG_PPC_64K_PAGES
247	if (ctblk_cache)
248		kmem_cache_destroy(ctblk_cache);
249#endif
250}
251
252#define EHCA_HCAAVER  EHCA_BMASK_IBM(32, 39)
253#define EHCA_REVID    EHCA_BMASK_IBM(40, 63)
254
255static struct cap_descr {
256	u64 mask;
257	char *descr;
258} hca_cap_descr[] = {
259	{ HCA_CAP_AH_PORT_NR_CHECK, "HCA_CAP_AH_PORT_NR_CHECK" },
260	{ HCA_CAP_ATOMIC, "HCA_CAP_ATOMIC" },
261	{ HCA_CAP_AUTO_PATH_MIG, "HCA_CAP_AUTO_PATH_MIG" },
262	{ HCA_CAP_BAD_P_KEY_CTR, "HCA_CAP_BAD_P_KEY_CTR" },
263	{ HCA_CAP_SQD_RTS_PORT_CHANGE, "HCA_CAP_SQD_RTS_PORT_CHANGE" },
264	{ HCA_CAP_CUR_QP_STATE_MOD, "HCA_CAP_CUR_QP_STATE_MOD" },
265	{ HCA_CAP_INIT_TYPE, "HCA_CAP_INIT_TYPE" },
266	{ HCA_CAP_PORT_ACTIVE_EVENT, "HCA_CAP_PORT_ACTIVE_EVENT" },
267	{ HCA_CAP_Q_KEY_VIOL_CTR, "HCA_CAP_Q_KEY_VIOL_CTR" },
268	{ HCA_CAP_WQE_RESIZE, "HCA_CAP_WQE_RESIZE" },
269	{ HCA_CAP_RAW_PACKET_MCAST, "HCA_CAP_RAW_PACKET_MCAST" },
270	{ HCA_CAP_SHUTDOWN_PORT, "HCA_CAP_SHUTDOWN_PORT" },
271	{ HCA_CAP_RC_LL_QP, "HCA_CAP_RC_LL_QP" },
272	{ HCA_CAP_SRQ, "HCA_CAP_SRQ" },
273	{ HCA_CAP_UD_LL_QP, "HCA_CAP_UD_LL_QP" },
274	{ HCA_CAP_RESIZE_MR, "HCA_CAP_RESIZE_MR" },
275	{ HCA_CAP_MINI_QP, "HCA_CAP_MINI_QP" },
276	{ HCA_CAP_H_ALLOC_RES_SYNC, "HCA_CAP_H_ALLOC_RES_SYNC" },
277};
278
279static int ehca_sense_attributes(struct ehca_shca *shca)
280{
281	int i, ret = 0;
282	u64 h_ret;
283	struct hipz_query_hca *rblock;
284	struct hipz_query_port *port;
285	const char *loc_code;
286
287	static const u32 pgsize_map[] = {
288		HCA_CAP_MR_PGSIZE_4K,  0x1000,
289		HCA_CAP_MR_PGSIZE_64K, 0x10000,
290		HCA_CAP_MR_PGSIZE_1M,  0x100000,
291		HCA_CAP_MR_PGSIZE_16M, 0x1000000,
292	};
293
294	ehca_gen_dbg("Probing adapter %s...",
295		     shca->ofdev->dev.of_node->full_name);
296	loc_code = of_get_property(shca->ofdev->dev.of_node, "ibm,loc-code",
297				   NULL);
298	if (loc_code)
299		ehca_gen_dbg(" ... location lode=%s", loc_code);
300
301	rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
302	if (!rblock) {
303		ehca_gen_err("Cannot allocate rblock memory.");
304		return -ENOMEM;
305	}
306
307	h_ret = hipz_h_query_hca(shca->ipz_hca_handle, rblock);
308	if (h_ret != H_SUCCESS) {
309		ehca_gen_err("Cannot query device properties. h_ret=%lli",
310			     h_ret);
311		ret = -EPERM;
312		goto sense_attributes1;
313	}
314
315	if (ehca_nr_ports == 1)
316		shca->num_ports = 1;
317	else
318		shca->num_ports = (u8)rblock->num_ports;
319
320	ehca_gen_dbg(" ... found %x ports", rblock->num_ports);
321
322	if (ehca_hw_level == 0) {
323		u32 hcaaver;
324		u32 revid;
325
326		hcaaver = EHCA_BMASK_GET(EHCA_HCAAVER, rblock->hw_ver);
327		revid   = EHCA_BMASK_GET(EHCA_REVID, rblock->hw_ver);
328
329		ehca_gen_dbg(" ... hardware version=%x:%x", hcaaver, revid);
330
331		if (hcaaver == 1) {
332			if (revid <= 3)
333				shca->hw_level = 0x10 | (revid + 1);
334			else
335				shca->hw_level = 0x14;
336		} else if (hcaaver == 2) {
337			if (revid == 0)
338				shca->hw_level = 0x21;
339			else if (revid == 0x10)
340				shca->hw_level = 0x22;
341			else if (revid == 0x20 || revid == 0x21)
342				shca->hw_level = 0x23;
343		}
344
345		if (!shca->hw_level) {
346			ehca_gen_warn("unknown hardware version"
347				      " - assuming default level");
348			shca->hw_level = 0x22;
349		}
350	} else
351		shca->hw_level = ehca_hw_level;
352	ehca_gen_dbg(" ... hardware level=%x", shca->hw_level);
353
354	shca->hca_cap = rblock->hca_cap_indicators;
355	ehca_gen_dbg(" ... HCA capabilities:");
356	for (i = 0; i < ARRAY_SIZE(hca_cap_descr); i++)
357		if (EHCA_BMASK_GET(hca_cap_descr[i].mask, shca->hca_cap))
358			ehca_gen_dbg("   %s", hca_cap_descr[i].descr);
359
360	/* Autodetect hCall locking -- the "H_ALLOC_RESOURCE synced" flag is
361	 * a firmware property, so it's valid across all adapters
362	 */
363	if (ehca_lock_hcalls == -1)
364		ehca_lock_hcalls = !EHCA_BMASK_GET(HCA_CAP_H_ALLOC_RES_SYNC,
365					shca->hca_cap);
366
367	/* translate supported MR page sizes; always support 4K */
368	shca->hca_cap_mr_pgsize = EHCA_PAGESIZE;
369	for (i = 0; i < ARRAY_SIZE(pgsize_map); i += 2)
370		if (rblock->memory_page_size_supported & pgsize_map[i])
371			shca->hca_cap_mr_pgsize |= pgsize_map[i + 1];
372
373	/* Set maximum number of CQs and QPs to calculate EQ size */
374	if (shca->max_num_qps == -1)
375		shca->max_num_qps = min_t(int, rblock->max_qp,
376					  EHCA_MAX_NUM_QUEUES);
377	else if (shca->max_num_qps < 1 || shca->max_num_qps > rblock->max_qp) {
378		ehca_gen_warn("The requested number of QPs is out of range "
379			      "(1 - %i) specified by HW. Value is set to %i",
380			      rblock->max_qp, rblock->max_qp);
381		shca->max_num_qps = rblock->max_qp;
382	}
383
384	if (shca->max_num_cqs == -1)
385		shca->max_num_cqs = min_t(int, rblock->max_cq,
386					  EHCA_MAX_NUM_QUEUES);
387	else if (shca->max_num_cqs < 1 || shca->max_num_cqs > rblock->max_cq) {
388		ehca_gen_warn("The requested number of CQs is out of range "
389			      "(1 - %i) specified by HW. Value is set to %i",
390			      rblock->max_cq, rblock->max_cq);
391	}
392
393	/* query max MTU from first port -- it's the same for all ports */
394	port = (struct hipz_query_port *)rblock;
395	h_ret = hipz_h_query_port(shca->ipz_hca_handle, 1, port);
396	if (h_ret != H_SUCCESS) {
397		ehca_gen_err("Cannot query port properties. h_ret=%lli",
398			     h_ret);
399		ret = -EPERM;
400		goto sense_attributes1;
401	}
402
403	shca->max_mtu = port->max_mtu;
404
405sense_attributes1:
406	ehca_free_fw_ctrlblock(rblock);
407	return ret;
408}
409
410static int init_node_guid(struct ehca_shca *shca)
411{
412	int ret = 0;
413	struct hipz_query_hca *rblock;
414
415	rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
416	if (!rblock) {
417		ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
418		return -ENOMEM;
419	}
420
421	if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
422		ehca_err(&shca->ib_device, "Can't query device properties");
423		ret = -EINVAL;
424		goto init_node_guid1;
425	}
426
427	memcpy(&shca->ib_device.node_guid, &rblock->node_guid, sizeof(u64));
428
429init_node_guid1:
430	ehca_free_fw_ctrlblock(rblock);
431	return ret;
432}
433
434static int ehca_init_device(struct ehca_shca *shca)
435{
436	int ret;
437
438	ret = init_node_guid(shca);
439	if (ret)
440		return ret;
441
442	strlcpy(shca->ib_device.name, "ehca%d", IB_DEVICE_NAME_MAX);
443	shca->ib_device.owner               = THIS_MODULE;
444
445	shca->ib_device.uverbs_abi_ver	    = 8;
446	shca->ib_device.uverbs_cmd_mask	    =
447		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT)		|
448		(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)	|
449		(1ull << IB_USER_VERBS_CMD_QUERY_PORT)		|
450		(1ull << IB_USER_VERBS_CMD_ALLOC_PD)		|
451		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD)		|
452		(1ull << IB_USER_VERBS_CMD_REG_MR)		|
453		(1ull << IB_USER_VERBS_CMD_DEREG_MR)		|
454		(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)	|
455		(1ull << IB_USER_VERBS_CMD_CREATE_CQ)		|
456		(1ull << IB_USER_VERBS_CMD_DESTROY_CQ)		|
457		(1ull << IB_USER_VERBS_CMD_CREATE_QP)		|
458		(1ull << IB_USER_VERBS_CMD_MODIFY_QP)		|
459		(1ull << IB_USER_VERBS_CMD_QUERY_QP)		|
460		(1ull << IB_USER_VERBS_CMD_DESTROY_QP)		|
461		(1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)	|
462		(1ull << IB_USER_VERBS_CMD_DETACH_MCAST);
463
464	shca->ib_device.node_type           = RDMA_NODE_IB_CA;
465	shca->ib_device.phys_port_cnt       = shca->num_ports;
466	shca->ib_device.num_comp_vectors    = 1;
467	shca->ib_device.dma_device          = &shca->ofdev->dev;
468	shca->ib_device.query_device        = ehca_query_device;
469	shca->ib_device.query_port          = ehca_query_port;
470	shca->ib_device.query_gid           = ehca_query_gid;
471	shca->ib_device.query_pkey          = ehca_query_pkey;
472	/* shca->in_device.modify_device    = ehca_modify_device    */
473	shca->ib_device.modify_port         = ehca_modify_port;
474	shca->ib_device.alloc_ucontext      = ehca_alloc_ucontext;
475	shca->ib_device.dealloc_ucontext    = ehca_dealloc_ucontext;
476	shca->ib_device.alloc_pd            = ehca_alloc_pd;
477	shca->ib_device.dealloc_pd          = ehca_dealloc_pd;
478	shca->ib_device.create_ah	    = ehca_create_ah;
479	/* shca->ib_device.modify_ah	    = ehca_modify_ah;	    */
480	shca->ib_device.query_ah	    = ehca_query_ah;
481	shca->ib_device.destroy_ah	    = ehca_destroy_ah;
482	shca->ib_device.create_qp	    = ehca_create_qp;
483	shca->ib_device.modify_qp	    = ehca_modify_qp;
484	shca->ib_device.query_qp	    = ehca_query_qp;
485	shca->ib_device.destroy_qp	    = ehca_destroy_qp;
486	shca->ib_device.post_send	    = ehca_post_send;
487	shca->ib_device.post_recv	    = ehca_post_recv;
488	shca->ib_device.create_cq	    = ehca_create_cq;
489	shca->ib_device.destroy_cq	    = ehca_destroy_cq;
490	shca->ib_device.resize_cq	    = ehca_resize_cq;
491	shca->ib_device.poll_cq		    = ehca_poll_cq;
492	/* shca->ib_device.peek_cq	    = ehca_peek_cq;	    */
493	shca->ib_device.req_notify_cq	    = ehca_req_notify_cq;
494	/* shca->ib_device.req_ncomp_notif  = ehca_req_ncomp_notif; */
495	shca->ib_device.get_dma_mr	    = ehca_get_dma_mr;
496	shca->ib_device.reg_phys_mr	    = ehca_reg_phys_mr;
497	shca->ib_device.reg_user_mr	    = ehca_reg_user_mr;
498	shca->ib_device.query_mr	    = ehca_query_mr;
499	shca->ib_device.dereg_mr	    = ehca_dereg_mr;
500	shca->ib_device.rereg_phys_mr	    = ehca_rereg_phys_mr;
501	shca->ib_device.alloc_mw	    = ehca_alloc_mw;
502	shca->ib_device.bind_mw		    = ehca_bind_mw;
503	shca->ib_device.dealloc_mw	    = ehca_dealloc_mw;
504	shca->ib_device.alloc_fmr	    = ehca_alloc_fmr;
505	shca->ib_device.map_phys_fmr	    = ehca_map_phys_fmr;
506	shca->ib_device.unmap_fmr	    = ehca_unmap_fmr;
507	shca->ib_device.dealloc_fmr	    = ehca_dealloc_fmr;
508	shca->ib_device.attach_mcast	    = ehca_attach_mcast;
509	shca->ib_device.detach_mcast	    = ehca_detach_mcast;
510	shca->ib_device.process_mad	    = ehca_process_mad;
511	shca->ib_device.mmap		    = ehca_mmap;
512	shca->ib_device.dma_ops		    = &ehca_dma_mapping_ops;
513
514	if (EHCA_BMASK_GET(HCA_CAP_SRQ, shca->hca_cap)) {
515		shca->ib_device.uverbs_cmd_mask |=
516			(1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
517			(1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
518			(1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
519			(1ull << IB_USER_VERBS_CMD_DESTROY_SRQ);
520
521		shca->ib_device.create_srq          = ehca_create_srq;
522		shca->ib_device.modify_srq          = ehca_modify_srq;
523		shca->ib_device.query_srq           = ehca_query_srq;
524		shca->ib_device.destroy_srq         = ehca_destroy_srq;
525		shca->ib_device.post_srq_recv       = ehca_post_srq_recv;
526	}
527
528	return ret;
529}
530
531static int ehca_create_aqp1(struct ehca_shca *shca, u32 port)
532{
533	struct ehca_sport *sport = &shca->sport[port - 1];
534	struct ib_cq *ibcq;
535	struct ib_qp *ibqp;
536	struct ib_qp_init_attr qp_init_attr;
537	int ret;
538
539	if (sport->ibcq_aqp1) {
540		ehca_err(&shca->ib_device, "AQP1 CQ is already created.");
541		return -EPERM;
542	}
543
544	ibcq = ib_create_cq(&shca->ib_device, NULL, NULL, (void *)(-1), 10, 0);
545	if (IS_ERR(ibcq)) {
546		ehca_err(&shca->ib_device, "Cannot create AQP1 CQ.");
547		return PTR_ERR(ibcq);
548	}
549	sport->ibcq_aqp1 = ibcq;
550
551	if (sport->ibqp_sqp[IB_QPT_GSI]) {
552		ehca_err(&shca->ib_device, "AQP1 QP is already created.");
553		ret = -EPERM;
554		goto create_aqp1;
555	}
556
557	memset(&qp_init_attr, 0, sizeof(struct ib_qp_init_attr));
558	qp_init_attr.send_cq          = ibcq;
559	qp_init_attr.recv_cq          = ibcq;
560	qp_init_attr.sq_sig_type      = IB_SIGNAL_ALL_WR;
561	qp_init_attr.cap.max_send_wr  = 100;
562	qp_init_attr.cap.max_recv_wr  = 100;
563	qp_init_attr.cap.max_send_sge = 2;
564	qp_init_attr.cap.max_recv_sge = 1;
565	qp_init_attr.qp_type          = IB_QPT_GSI;
566	qp_init_attr.port_num         = port;
567	qp_init_attr.qp_context       = NULL;
568	qp_init_attr.event_handler    = NULL;
569	qp_init_attr.srq              = NULL;
570
571	ibqp = ib_create_qp(&shca->pd->ib_pd, &qp_init_attr);
572	if (IS_ERR(ibqp)) {
573		ehca_err(&shca->ib_device, "Cannot create AQP1 QP.");
574		ret = PTR_ERR(ibqp);
575		goto create_aqp1;
576	}
577	sport->ibqp_sqp[IB_QPT_GSI] = ibqp;
578
579	return 0;
580
581create_aqp1:
582	ib_destroy_cq(sport->ibcq_aqp1);
583	return ret;
584}
585
586static int ehca_destroy_aqp1(struct ehca_sport *sport)
587{
588	int ret;
589
590	ret = ib_destroy_qp(sport->ibqp_sqp[IB_QPT_GSI]);
591	if (ret) {
592		ehca_gen_err("Cannot destroy AQP1 QP. ret=%i", ret);
593		return ret;
594	}
595
596	ret = ib_destroy_cq(sport->ibcq_aqp1);
597	if (ret)
598		ehca_gen_err("Cannot destroy AQP1 CQ. ret=%i", ret);
599
600	return ret;
601}
602
603static ssize_t ehca_show_debug_level(struct device_driver *ddp, char *buf)
604{
605	return snprintf(buf, PAGE_SIZE, "%d\n", ehca_debug_level);
606}
607
608static ssize_t ehca_store_debug_level(struct device_driver *ddp,
609				      const char *buf, size_t count)
610{
611	int value = (*buf) - '0';
612	if (value >= 0 && value <= 9)
613		ehca_debug_level = value;
614	return 1;
615}
616
617static DRIVER_ATTR(debug_level, S_IRUSR | S_IWUSR,
618		   ehca_show_debug_level, ehca_store_debug_level);
619
620static struct attribute *ehca_drv_attrs[] = {
621	&driver_attr_debug_level.attr,
622	NULL
623};
624
625static struct attribute_group ehca_drv_attr_grp = {
626	.attrs = ehca_drv_attrs
627};
628
629static const struct attribute_group *ehca_drv_attr_groups[] = {
630	&ehca_drv_attr_grp,
631	NULL,
632};
633
634#define EHCA_RESOURCE_ATTR(name)                                           \
635static ssize_t  ehca_show_##name(struct device *dev,                       \
636				 struct device_attribute *attr,            \
637				 char *buf)                                \
638{									   \
639	struct ehca_shca *shca;						   \
640	struct hipz_query_hca *rblock;				           \
641	int data;                                                          \
642									   \
643	shca = dev_get_drvdata(dev);					   \
644									   \
645	rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);			   \
646	if (!rblock) {						           \
647		dev_err(dev, "Can't allocate rblock memory.\n");           \
648		return 0;						   \
649	}								   \
650									   \
651	if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) { \
652		dev_err(dev, "Can't query device properties\n");           \
653		ehca_free_fw_ctrlblock(rblock);			   	   \
654		return 0;					   	   \
655	}								   \
656									   \
657	data = rblock->name;                                               \
658	ehca_free_fw_ctrlblock(rblock);                                    \
659									   \
660	if ((strcmp(#name, "num_ports") == 0) && (ehca_nr_ports == 1))	   \
661		return snprintf(buf, 256, "1\n");			   \
662	else								   \
663		return snprintf(buf, 256, "%d\n", data);		   \
664									   \
665}									   \
666static DEVICE_ATTR(name, S_IRUGO, ehca_show_##name, NULL);
667
668EHCA_RESOURCE_ATTR(num_ports);
669EHCA_RESOURCE_ATTR(hw_ver);
670EHCA_RESOURCE_ATTR(max_eq);
671EHCA_RESOURCE_ATTR(cur_eq);
672EHCA_RESOURCE_ATTR(max_cq);
673EHCA_RESOURCE_ATTR(cur_cq);
674EHCA_RESOURCE_ATTR(max_qp);
675EHCA_RESOURCE_ATTR(cur_qp);
676EHCA_RESOURCE_ATTR(max_mr);
677EHCA_RESOURCE_ATTR(cur_mr);
678EHCA_RESOURCE_ATTR(max_mw);
679EHCA_RESOURCE_ATTR(cur_mw);
680EHCA_RESOURCE_ATTR(max_pd);
681EHCA_RESOURCE_ATTR(max_ah);
682
683static ssize_t ehca_show_adapter_handle(struct device *dev,
684					struct device_attribute *attr,
685					char *buf)
686{
687	struct ehca_shca *shca = dev_get_drvdata(dev);
688
689	return sprintf(buf, "%llx\n", shca->ipz_hca_handle.handle);
690
691}
692static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL);
693
694static struct attribute *ehca_dev_attrs[] = {
695	&dev_attr_adapter_handle.attr,
696	&dev_attr_num_ports.attr,
697	&dev_attr_hw_ver.attr,
698	&dev_attr_max_eq.attr,
699	&dev_attr_cur_eq.attr,
700	&dev_attr_max_cq.attr,
701	&dev_attr_cur_cq.attr,
702	&dev_attr_max_qp.attr,
703	&dev_attr_cur_qp.attr,
704	&dev_attr_max_mr.attr,
705	&dev_attr_cur_mr.attr,
706	&dev_attr_max_mw.attr,
707	&dev_attr_cur_mw.attr,
708	&dev_attr_max_pd.attr,
709	&dev_attr_max_ah.attr,
710	NULL
711};
712
713static struct attribute_group ehca_dev_attr_grp = {
714	.attrs = ehca_dev_attrs
715};
716
717static int ehca_probe(struct platform_device *dev)
718{
719	struct ehca_shca *shca;
720	const u64 *handle;
721	struct ib_pd *ibpd;
722	int ret, i, eq_size;
723	unsigned long flags;
724
725	handle = of_get_property(dev->dev.of_node, "ibm,hca-handle", NULL);
726	if (!handle) {
727		ehca_gen_err("Cannot get eHCA handle for adapter: %s.",
728			     dev->dev.of_node->full_name);
729		return -ENODEV;
730	}
731
732	if (!(*handle)) {
733		ehca_gen_err("Wrong eHCA handle for adapter: %s.",
734			     dev->dev.of_node->full_name);
735		return -ENODEV;
736	}
737
738	shca = (struct ehca_shca *)ib_alloc_device(sizeof(*shca));
739	if (!shca) {
740		ehca_gen_err("Cannot allocate shca memory.");
741		return -ENOMEM;
742	}
743
744	mutex_init(&shca->modify_mutex);
745	atomic_set(&shca->num_cqs, 0);
746	atomic_set(&shca->num_qps, 0);
747	shca->max_num_qps = ehca_max_qp;
748	shca->max_num_cqs = ehca_max_cq;
749
750	for (i = 0; i < ARRAY_SIZE(shca->sport); i++)
751		spin_lock_init(&shca->sport[i].mod_sqp_lock);
752
753	shca->ofdev = dev;
754	shca->ipz_hca_handle.handle = *handle;
755	dev_set_drvdata(&dev->dev, shca);
756
757	ret = ehca_sense_attributes(shca);
758	if (ret < 0) {
759		ehca_gen_err("Cannot sense eHCA attributes.");
760		goto probe1;
761	}
762
763	ret = ehca_init_device(shca);
764	if (ret) {
765		ehca_gen_err("Cannot init ehca  device struct");
766		goto probe1;
767	}
768
769	eq_size = 2 * shca->max_num_cqs + 4 * shca->max_num_qps;
770	/* create event queues */
771	ret = ehca_create_eq(shca, &shca->eq, EHCA_EQ, eq_size);
772	if (ret) {
773		ehca_err(&shca->ib_device, "Cannot create EQ.");
774		goto probe1;
775	}
776
777	ret = ehca_create_eq(shca, &shca->neq, EHCA_NEQ, 513);
778	if (ret) {
779		ehca_err(&shca->ib_device, "Cannot create NEQ.");
780		goto probe3;
781	}
782
783	/* create internal protection domain */
784	ibpd = ehca_alloc_pd(&shca->ib_device, (void *)(-1), NULL);
785	if (IS_ERR(ibpd)) {
786		ehca_err(&shca->ib_device, "Cannot create internal PD.");
787		ret = PTR_ERR(ibpd);
788		goto probe4;
789	}
790
791	shca->pd = container_of(ibpd, struct ehca_pd, ib_pd);
792	shca->pd->ib_pd.device = &shca->ib_device;
793
794	/* create internal max MR */
795	ret = ehca_reg_internal_maxmr(shca, shca->pd, &shca->maxmr);
796
797	if (ret) {
798		ehca_err(&shca->ib_device, "Cannot create internal MR ret=%i",
799			 ret);
800		goto probe5;
801	}
802
803	ret = ib_register_device(&shca->ib_device, NULL);
804	if (ret) {
805		ehca_err(&shca->ib_device,
806			 "ib_register_device() failed ret=%i", ret);
807		goto probe6;
808	}
809
810	/* create AQP1 for port 1 */
811	if (ehca_open_aqp1 == 1) {
812		shca->sport[0].port_state = IB_PORT_DOWN;
813		ret = ehca_create_aqp1(shca, 1);
814		if (ret) {
815			ehca_err(&shca->ib_device,
816				 "Cannot create AQP1 for port 1.");
817			goto probe7;
818		}
819	}
820
821	/* create AQP1 for port 2 */
822	if ((ehca_open_aqp1 == 1) && (shca->num_ports == 2)) {
823		shca->sport[1].port_state = IB_PORT_DOWN;
824		ret = ehca_create_aqp1(shca, 2);
825		if (ret) {
826			ehca_err(&shca->ib_device,
827				 "Cannot create AQP1 for port 2.");
828			goto probe8;
829		}
830	}
831
832	ret = sysfs_create_group(&dev->dev.kobj, &ehca_dev_attr_grp);
833	if (ret) /* only complain; we can live without attributes */
834		ehca_err(&shca->ib_device,
835			 "Cannot create device attributes  ret=%d", ret);
836
837	spin_lock_irqsave(&shca_list_lock, flags);
838	list_add(&shca->shca_list, &shca_list);
839	spin_unlock_irqrestore(&shca_list_lock, flags);
840
841	return 0;
842
843probe8:
844	ret = ehca_destroy_aqp1(&shca->sport[0]);
845	if (ret)
846		ehca_err(&shca->ib_device,
847			 "Cannot destroy AQP1 for port 1. ret=%i", ret);
848
849probe7:
850	ib_unregister_device(&shca->ib_device);
851
852probe6:
853	ret = ehca_dereg_internal_maxmr(shca);
854	if (ret)
855		ehca_err(&shca->ib_device,
856			 "Cannot destroy internal MR. ret=%x", ret);
857
858probe5:
859	ret = ehca_dealloc_pd(&shca->pd->ib_pd);
860	if (ret)
861		ehca_err(&shca->ib_device,
862			 "Cannot destroy internal PD. ret=%x", ret);
863
864probe4:
865	ret = ehca_destroy_eq(shca, &shca->neq);
866	if (ret)
867		ehca_err(&shca->ib_device,
868			 "Cannot destroy NEQ. ret=%x", ret);
869
870probe3:
871	ret = ehca_destroy_eq(shca, &shca->eq);
872	if (ret)
873		ehca_err(&shca->ib_device,
874			 "Cannot destroy EQ. ret=%x", ret);
875
876probe1:
877	ib_dealloc_device(&shca->ib_device);
878
879	return -EINVAL;
880}
881
882static int ehca_remove(struct platform_device *dev)
883{
884	struct ehca_shca *shca = dev_get_drvdata(&dev->dev);
885	unsigned long flags;
886	int ret;
887
888	sysfs_remove_group(&dev->dev.kobj, &ehca_dev_attr_grp);
889
890	if (ehca_open_aqp1 == 1) {
891		int i;
892		for (i = 0; i < shca->num_ports; i++) {
893			ret = ehca_destroy_aqp1(&shca->sport[i]);
894			if (ret)
895				ehca_err(&shca->ib_device,
896					 "Cannot destroy AQP1 for port %x "
897					 "ret=%i", ret, i);
898		}
899	}
900
901	ib_unregister_device(&shca->ib_device);
902
903	ret = ehca_dereg_internal_maxmr(shca);
904	if (ret)
905		ehca_err(&shca->ib_device,
906			 "Cannot destroy internal MR. ret=%i", ret);
907
908	ret = ehca_dealloc_pd(&shca->pd->ib_pd);
909	if (ret)
910		ehca_err(&shca->ib_device,
911			 "Cannot destroy internal PD. ret=%i", ret);
912
913	ret = ehca_destroy_eq(shca, &shca->eq);
914	if (ret)
915		ehca_err(&shca->ib_device, "Cannot destroy EQ. ret=%i", ret);
916
917	ret = ehca_destroy_eq(shca, &shca->neq);
918	if (ret)
919		ehca_err(&shca->ib_device, "Canot destroy NEQ. ret=%i", ret);
920
921	ib_dealloc_device(&shca->ib_device);
922
923	spin_lock_irqsave(&shca_list_lock, flags);
924	list_del(&shca->shca_list);
925	spin_unlock_irqrestore(&shca_list_lock, flags);
926
927	return ret;
928}
929
930static struct of_device_id ehca_device_table[] =
931{
932	{
933		.name       = "lhca",
934		.compatible = "IBM,lhca",
935	},
936	{},
937};
938MODULE_DEVICE_TABLE(of, ehca_device_table);
939
940static struct platform_driver ehca_driver = {
941	.probe       = ehca_probe,
942	.remove      = ehca_remove,
943	.driver = {
944		.name = "ehca",
945		.owner = THIS_MODULE,
946		.groups = ehca_drv_attr_groups,
947		.of_match_table = ehca_device_table,
948	},
949};
950
951void ehca_poll_eqs(unsigned long data)
952{
953	struct ehca_shca *shca;
954
955	spin_lock(&shca_list_lock);
956	list_for_each_entry(shca, &shca_list, shca_list) {
957		if (shca->eq.is_initialized) {
958			/* call deadman proc only if eq ptr does not change */
959			struct ehca_eq *eq = &shca->eq;
960			int max = 3;
961			volatile u64 q_ofs, q_ofs2;
962			unsigned long flags;
963			spin_lock_irqsave(&eq->spinlock, flags);
964			q_ofs = eq->ipz_queue.current_q_offset;
965			spin_unlock_irqrestore(&eq->spinlock, flags);
966			do {
967				spin_lock_irqsave(&eq->spinlock, flags);
968				q_ofs2 = eq->ipz_queue.current_q_offset;
969				spin_unlock_irqrestore(&eq->spinlock, flags);
970				max--;
971			} while (q_ofs == q_ofs2 && max > 0);
972			if (q_ofs == q_ofs2)
973				ehca_process_eq(shca, 0);
974		}
975	}
976	mod_timer(&poll_eqs_timer, round_jiffies(jiffies + HZ));
977	spin_unlock(&shca_list_lock);
978}
979
980static int ehca_mem_notifier(struct notifier_block *nb,
981			     unsigned long action, void *data)
982{
983	static unsigned long ehca_dmem_warn_time;
984	unsigned long flags;
985
986	switch (action) {
987	case MEM_CANCEL_OFFLINE:
988	case MEM_CANCEL_ONLINE:
989	case MEM_ONLINE:
990	case MEM_OFFLINE:
991		return NOTIFY_OK;
992	case MEM_GOING_ONLINE:
993	case MEM_GOING_OFFLINE:
994		/* only ok if no hca is attached to the lpar */
995		spin_lock_irqsave(&shca_list_lock, flags);
996		if (list_empty(&shca_list)) {
997			spin_unlock_irqrestore(&shca_list_lock, flags);
998			return NOTIFY_OK;
999		} else {
1000			spin_unlock_irqrestore(&shca_list_lock, flags);
1001			if (printk_timed_ratelimit(&ehca_dmem_warn_time,
1002						   30 * 1000))
1003				ehca_gen_err("DMEM operations are not allowed"
1004					     "in conjunction with eHCA");
1005			return NOTIFY_BAD;
1006		}
1007	}
1008	return NOTIFY_OK;
1009}
1010
1011static struct notifier_block ehca_mem_nb = {
1012	.notifier_call = ehca_mem_notifier,
1013};
1014
1015static int __init ehca_module_init(void)
1016{
1017	int ret;
1018
1019	printk(KERN_INFO "eHCA Infiniband Device Driver "
1020	       "(Version " HCAD_VERSION ")\n");
1021
1022	ret = ehca_create_comp_pool();
1023	if (ret) {
1024		ehca_gen_err("Cannot create comp pool.");
1025		return ret;
1026	}
1027
1028	ret = ehca_create_slab_caches();
1029	if (ret) {
1030		ehca_gen_err("Cannot create SLAB caches");
1031		ret = -ENOMEM;
1032		goto module_init1;
1033	}
1034
1035	ret = ehca_create_busmap();
1036	if (ret) {
1037		ehca_gen_err("Cannot create busmap.");
1038		goto module_init2;
1039	}
1040
1041	ret = ibmebus_register_driver(&ehca_driver);
1042	if (ret) {
1043		ehca_gen_err("Cannot register eHCA device driver");
1044		ret = -EINVAL;
1045		goto module_init3;
1046	}
1047
1048	ret = register_memory_notifier(&ehca_mem_nb);
1049	if (ret) {
1050		ehca_gen_err("Failed registering memory add/remove notifier");
1051		goto module_init4;
1052	}
1053
1054	if (ehca_poll_all_eqs != 1) {
1055		ehca_gen_err("WARNING!!!");
1056		ehca_gen_err("It is possible to lose interrupts.");
1057	} else {
1058		init_timer(&poll_eqs_timer);
1059		poll_eqs_timer.function = ehca_poll_eqs;
1060		poll_eqs_timer.expires = jiffies + HZ;
1061		add_timer(&poll_eqs_timer);
1062	}
1063
1064	return 0;
1065
1066module_init4:
1067	ibmebus_unregister_driver(&ehca_driver);
1068
1069module_init3:
1070	ehca_destroy_busmap();
1071
1072module_init2:
1073	ehca_destroy_slab_caches();
1074
1075module_init1:
1076	ehca_destroy_comp_pool();
1077	return ret;
1078};
1079
1080static void __exit ehca_module_exit(void)
1081{
1082	if (ehca_poll_all_eqs == 1)
1083		del_timer_sync(&poll_eqs_timer);
1084
1085	ibmebus_unregister_driver(&ehca_driver);
1086
1087	unregister_memory_notifier(&ehca_mem_nb);
1088
1089	ehca_destroy_busmap();
1090
1091	ehca_destroy_slab_caches();
1092
1093	ehca_destroy_comp_pool();
1094
1095	idr_destroy(&ehca_cq_idr);
1096	idr_destroy(&ehca_qp_idr);
1097};
1098
1099module_init(ehca_module_init);
1100module_exit(ehca_module_exit);
1101