1/* Copyright (C) 2010 - 2013 UNISYS CORPORATION
2 * All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or (at
7 * your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
12 * NON INFRINGEMENT.  See the GNU General Public License for more
13 * details.
14 */
15
16#ifndef __CHANNEL_H__
17#define __CHANNEL_H__
18
19#include <linux/types.h>
20#include <linux/io.h>
21#include <linux/uuid.h>
22
23/*
24* Whenever this file is changed a corresponding change must be made in
25* the Console/ServicePart/visordiag_early/supervisor_channel.h file
26* which is needed for Linux kernel compiles. These two files must be
27* in sync.
28*/
29
30/* define the following to prevent include nesting in kernel header
31 * files of similar abbreviated content
32 */
33#define __SUPERVISOR_CHANNEL_H__
34
35#define SIGNATURE_16(A, B) ((A) | (B << 8))
36#define SIGNATURE_32(A, B, C, D) \
37	(SIGNATURE_16(A, B) | (SIGNATURE_16(C, D) << 16))
38#define SIGNATURE_64(A, B, C, D, E, F, G, H) \
39	(SIGNATURE_32(A, B, C, D) | ((u64)(SIGNATURE_32(E, F, G, H)) << 32))
40
41#ifndef lengthof
42#define lengthof(TYPE, MEMBER) (sizeof(((TYPE *)0)->MEMBER))
43#endif
44#ifndef COVERQ
45#define COVERQ(v, d)  (((v) + (d) - 1) / (d))
46#endif
47#ifndef COVER
48#define COVER(v, d)   ((d) * COVERQ(v, d))
49#endif
50
51#define ULTRA_CHANNEL_PROTOCOL_SIGNATURE  SIGNATURE_32('E', 'C', 'N', 'L')
52
53enum channel_serverstate {
54	CHANNELSRV_UNINITIALIZED = 0,	/* channel is in an undefined state */
55	CHANNELSRV_READY = 1	/* channel has been initialized by server */
56};
57
58enum channel_clientstate {
59	CHANNELCLI_DETACHED = 0,
60	CHANNELCLI_DISABLED = 1,	/* client can see channel but is NOT
61					 * allowed to use it unless given TBD
62					 * explicit request (should actually be
63					 * < DETACHED) */
64	CHANNELCLI_ATTACHING = 2,	/* legacy EFI client request
65					 * for EFI server to attach */
66	CHANNELCLI_ATTACHED = 3,	/* idle, but client may want
67					 * to use channel any time */
68	CHANNELCLI_BUSY = 4,	/* client either wants to use or is
69				 * using channel */
70	CHANNELCLI_OWNED = 5	/* "no worries" state - client can
71				 * access channel anytime */
72};
73
74static inline const u8 *
75ULTRA_CHANNELCLI_STRING(u32 v)
76{
77	switch (v) {
78	case CHANNELCLI_DETACHED:
79		return (const u8 *)("DETACHED");
80	case CHANNELCLI_DISABLED:
81		return (const u8 *)("DISABLED");
82	case CHANNELCLI_ATTACHING:
83		return (const u8 *)("ATTACHING");
84	case CHANNELCLI_ATTACHED:
85		return (const u8 *)("ATTACHED");
86	case CHANNELCLI_BUSY:
87		return (const u8 *)("BUSY");
88	case CHANNELCLI_OWNED:
89		return (const u8 *)("OWNED");
90	default:
91		break;
92	}
93	return (const u8 *)("?");
94}
95
96#define SPAR_CHANNEL_SERVER_READY(ch) \
97	(readl(&(ch)->srv_state) == CHANNELSRV_READY)
98
99#define ULTRA_VALID_CHANNELCLI_TRANSITION(o, n)				\
100	(((((o) == CHANNELCLI_DETACHED) && ((n) == CHANNELCLI_DISABLED)) || \
101	  (((o) == CHANNELCLI_ATTACHING) && ((n) == CHANNELCLI_DISABLED)) || \
102	  (((o) == CHANNELCLI_ATTACHED) && ((n) == CHANNELCLI_DISABLED)) || \
103	  (((o) == CHANNELCLI_ATTACHING) && ((n) == CHANNELCLI_DETACHED)) || \
104	  (((o) == CHANNELCLI_ATTACHED) && ((n) == CHANNELCLI_DETACHED)) || \
105	  (((o) == CHANNELCLI_DETACHED) && ((n) == CHANNELCLI_ATTACHING)) || \
106	  (((o) == CHANNELCLI_ATTACHING) && ((n) == CHANNELCLI_ATTACHED)) || \
107	  (((o) == CHANNELCLI_DETACHED) && ((n) == CHANNELCLI_ATTACHED)) || \
108	  (((o) == CHANNELCLI_BUSY) && ((n) == CHANNELCLI_ATTACHED)) ||	\
109	  (((o) == CHANNELCLI_ATTACHED) && ((n) == CHANNELCLI_BUSY)) ||	\
110	  (((o) == CHANNELCLI_DETACHED) && ((n) == CHANNELCLI_OWNED)) || \
111	  (((o) == CHANNELCLI_DISABLED) && ((n) == CHANNELCLI_OWNED)) || \
112	  (((o) == CHANNELCLI_ATTACHING) && ((n) == CHANNELCLI_OWNED)) || \
113	  (((o) == CHANNELCLI_ATTACHED) && ((n) == CHANNELCLI_OWNED)) || \
114	  (((o) == CHANNELCLI_BUSY) && ((n) == CHANNELCLI_OWNED)) || (0)) \
115	 ? (1) : (0))
116
117/* Values for ULTRA_CHANNEL_PROTOCOL.CliErrorBoot: */
118/* throttling invalid boot channel statetransition error due to client
119 * disabled */
120#define ULTRA_CLIERRORBOOT_THROTTLEMSG_DISABLED    0x01
121
122/* throttling invalid boot channel statetransition error due to client
123 * not attached */
124#define ULTRA_CLIERRORBOOT_THROTTLEMSG_NOTATTACHED 0x02
125
126/* throttling invalid boot channel statetransition error due to busy channel */
127#define ULTRA_CLIERRORBOOT_THROTTLEMSG_BUSY        0x04
128
129/* Values for ULTRA_CHANNEL_PROTOCOL.CliErrorOS: */
130/* throttling invalid guest OS channel statetransition error due to
131 * client disabled */
132#define ULTRA_CLIERROROS_THROTTLEMSG_DISABLED      0x01
133
134/* throttling invalid guest OS channel statetransition error due to
135 * client not attached */
136#define ULTRA_CLIERROROS_THROTTLEMSG_NOTATTACHED   0x02
137
138/* throttling invalid guest OS channel statetransition error due to
139 * busy channel */
140#define ULTRA_CLIERROROS_THROTTLEMSG_BUSY          0x04
141
142/* Values for ULTRA_CHANNEL_PROTOCOL.Features: This define exists so
143* that windows guest can look at the FeatureFlags in the io channel,
144* and configure the windows driver to use interrupts or not based on
145* this setting.  This flag is set in uislib after the
146* ULTRA_VHBA_init_channel is called.  All feature bits for all
147* channels should be defined here.  The io channel feature bits are
148* defined right here */
149#define ULTRA_IO_DRIVER_ENABLES_INTS (0x1ULL << 1)
150#define ULTRA_IO_CHANNEL_IS_POLLING (0x1ULL << 3)
151#define ULTRA_IO_IOVM_IS_OK_WITH_DRIVER_DISABLING_INTS (0x1ULL << 4)
152#define ULTRA_IO_DRIVER_DISABLES_INTS (0x1ULL << 5)
153#define ULTRA_IO_DRIVER_SUPPORTS_ENHANCED_RCVBUF_CHECKING (0x1ULL << 6)
154
155/* Common Channel Header */
156struct channel_header {
157	u64 signature;		/* Signature */
158	u32 legacy_state;	/* DEPRECATED - being replaced by */
159	/* /              SrvState, CliStateBoot, and CliStateOS below */
160	u32 header_size;	/* sizeof(struct channel_header) */
161	u64 size;		/* Total size of this channel in bytes */
162	u64 features;		/* Flags to modify behavior */
163	uuid_le chtype;		/* Channel type: data, bus, control, etc. */
164	u64 partition_handle;	/* ID of guest partition */
165	u64 handle;		/* Device number of this channel in client */
166	u64 ch_space_offset;	/* Offset in bytes to channel specific area */
167	u32 version_id;		/* struct channel_header Version ID */
168	u32 partition_index;	/* Index of guest partition */
169	uuid_le zone_uuid;	/* Guid of Channel's zone */
170	u32 cli_str_offset;	/* offset from channel header to
171				 * nul-terminated ClientString (0 if
172				 * ClientString not present) */
173	u32 cli_state_boot;	/* CHANNEL_CLIENTSTATE of pre-boot
174				 * EFI client of this channel */
175	u32 cmd_state_cli;	/* CHANNEL_COMMANDSTATE (overloaded in
176				 * Windows drivers, see ServerStateUp,
177				 * ServerStateDown, etc) */
178	u32 cli_state_os;	/* CHANNEL_CLIENTSTATE of Guest OS
179				 * client of this channel */
180	u32 ch_characteristic;	/* CHANNEL_CHARACTERISTIC_<xxx> */
181	u32 cmd_state_srv;	/* CHANNEL_COMMANDSTATE (overloaded in
182				 * Windows drivers, see ServerStateUp,
183				 * ServerStateDown, etc) */
184	u32 srv_state;		/* CHANNEL_SERVERSTATE */
185	u8 cli_error_boot;	/* bits to indicate err states for
186				 * boot clients, so err messages can
187				 * be throttled */
188	u8 cli_error_os;	/* bits to indicate err states for OS
189				 * clients, so err messages can be
190				 * throttled */
191	u8 filler[1];		/* Pad out to 128 byte cacheline */
192	/* Please add all new single-byte values below here */
193	u8 recover_channel;
194} __packed;
195
196#define ULTRA_CHANNEL_ENABLE_INTS (0x1ULL << 0)
197
198/* Subheader for the Signal Type variation of the Common Channel */
199struct signal_queue_header {
200	/* 1st cache line */
201	u32 version;		/* SIGNAL_QUEUE_HEADER Version ID */
202	u32 chtype;		/* Queue type: storage, network */
203	u64 size;		/* Total size of this queue in bytes */
204	u64 sig_base_offset;	/* Offset to signal queue area */
205	u64 features;		/* Flags to modify behavior */
206	u64 num_sent;		/* Total # of signals placed in this queue */
207	u64 num_overflows;	/* Total # of inserts failed due to
208				 * full queue */
209	u32 signal_size;	/* Total size of a signal for this queue */
210	u32 max_slots;		/* Max # of slots in queue, 1 slot is
211				 * always empty */
212	u32 max_signals;	/* Max # of signals in queue
213				 * (MaxSignalSlots-1) */
214	u32 head;		/* Queue head signal # */
215	/* 2nd cache line */
216	u64 num_received;	/* Total # of signals removed from this queue */
217	u32 tail;		/* Queue tail signal # (on separate
218				 * cache line) */
219	u32 reserved1;		/* Reserved field */
220	u64 reserved2;		/* Reserved field */
221	u64 client_queue;
222	u64 num_irq_received;	/* Total # of Interrupts received.  This
223					 * is incremented by the ISR in the
224					 * guest windows driver */
225	u64 num_empty;		/* Number of times that visor_signal_remove
226				 * is called and returned Empty
227				 * Status. */
228	u32 errorflags;		/* Error bits set during SignalReinit
229				 * to denote trouble with client's
230				 * fields */
231	u8 filler[12];		/* Pad out to 64 byte cacheline */
232} __packed;
233
234#define spar_signal_init(chan, QHDRFLD, QDATAFLD, QDATATYPE, ver, typ)	\
235	do {								\
236		memset(&chan->QHDRFLD, 0, sizeof(chan->QHDRFLD));	\
237		chan->QHDRFLD.version = ver;				\
238		chan->QHDRFLD.chtype = typ;				\
239		chan->QHDRFLD.size = sizeof(chan->QDATAFLD);		\
240		chan->QHDRFLD.signal_size = sizeof(QDATATYPE);		\
241		chan->QHDRFLD.sig_base_offset = (u64)(chan->QDATAFLD) -	\
242			(u64)(&chan->QHDRFLD);				\
243		chan->QHDRFLD.max_slots =				\
244			sizeof(chan->QDATAFLD) / sizeof(QDATATYPE);	\
245		chan->QHDRFLD.max_signals = chan->QHDRFLD.max_slots - 1;\
246	} while (0)
247
248/* Generic function useful for validating any type of channel when it is
249 * received by the client that will be accessing the channel.
250 * Note that <logCtx> is only needed for callers in the EFI environment, and
251 * is used to pass the EFI_DIAG_CAPTURE_PROTOCOL needed to log messages.
252 */
253static inline int
254spar_check_channel_client(void __iomem *ch,
255			  uuid_le expected_uuid,
256			  char *chname,
257			  u64 expected_min_bytes,
258			  u32 expected_version,
259			  u64 expected_signature)
260{
261	if (uuid_le_cmp(expected_uuid, NULL_UUID_LE) != 0) {
262		uuid_le guid;
263
264		memcpy_fromio(&guid,
265			      &((struct channel_header __iomem *)(ch))->chtype,
266			      sizeof(guid));
267		/* caller wants us to verify type GUID */
268		if (uuid_le_cmp(guid, expected_uuid) != 0) {
269			pr_err("Channel mismatch on channel=%s(%pUL) field=type expected=%pUL actual=%pUL\n",
270			       chname, &expected_uuid,
271			       &expected_uuid, &guid);
272			return 0;
273		}
274	}
275	if (expected_min_bytes > 0) {	/* caller wants us to verify
276					 * channel size */
277		unsigned long long bytes =
278				readq(&((struct channel_header __iomem *)
279					(ch))->size);
280		if (bytes < expected_min_bytes) {
281			pr_err("Channel mismatch on channel=%s(%pUL) field=size expected=0x%-8.8Lx actual=0x%-8.8Lx\n",
282			       chname, &expected_uuid,
283			       (unsigned long long)expected_min_bytes, bytes);
284			return 0;
285		}
286	}
287	if (expected_version > 0) {	/* caller wants us to verify
288					 * channel version */
289		unsigned long ver = readl(&((struct channel_header __iomem *)
290				    (ch))->version_id);
291		if (ver != expected_version) {
292			pr_err("Channel mismatch on channel=%s(%pUL) field=version expected=0x%-8.8lx actual=0x%-8.8lx\n",
293			       chname, &expected_uuid,
294			       (unsigned long)expected_version, ver);
295			return 0;
296		}
297	}
298	if (expected_signature > 0) {	/* caller wants us to verify
299					 * channel signature */
300		unsigned long long sig =
301				readq(&((struct channel_header __iomem *)
302					(ch))->signature);
303		if (sig != expected_signature) {
304			pr_err("Channel mismatch on channel=%s(%pUL) field=signature expected=0x%-8.8llx actual=0x%-8.8llx\n",
305			       chname, &expected_uuid,
306			       expected_signature, sig);
307			return 0;
308		}
309	}
310	return 1;
311}
312
313/* Generic function useful for validating any type of channel when it is about
314 * to be initialized by the server of the channel.
315 * Note that <logCtx> is only needed for callers in the EFI environment, and
316 * is used to pass the EFI_DIAG_CAPTURE_PROTOCOL needed to log messages.
317 */
318static inline int spar_check_channel_server(uuid_le typeuuid, char *name,
319					    u64 expected_min_bytes,
320					    u64 actual_bytes)
321{
322	if (expected_min_bytes > 0)	/* caller wants us to verify
323					 * channel size */
324		if (actual_bytes < expected_min_bytes) {
325			pr_err("Channel mismatch on channel=%s(%pUL) field=size expected=0x%-8.8llx actual=0x%-8.8llx\n",
326			       name, &typeuuid, expected_min_bytes,
327			       actual_bytes);
328			return 0;
329		}
330	return 1;
331}
332
333/* Given a file pathname <s> (with '/' or '\' separating directory nodes),
334 * returns a pointer to the beginning of a node within that pathname such
335 * that the number of nodes from that pointer to the end of the string is
336 * NOT more than <n>.  Note that if the pathname has less than <n> nodes
337 * in it, the return pointer will be to the beginning of the string.
338 */
339static inline u8 *
340pathname_last_n_nodes(u8 *s, unsigned int n)
341{
342	u8 *p = s;
343	unsigned int node_count = 0;
344
345	while (*p != '\0') {
346		if ((*p == '/') || (*p == '\\'))
347			node_count++;
348		p++;
349	}
350	if (node_count <= n)
351		return s;
352	while (n > 0) {
353		p--;
354		if (p == s)
355			break;	/* should never happen, unless someone
356				 * is changing the string while we are
357				 * looking at it!! */
358		if ((*p == '/') || (*p == '\\'))
359			n--;
360	}
361	return p + 1;
362}
363
364static inline int
365spar_channel_client_acquire_os(void __iomem *ch, u8 *id)
366{
367	struct channel_header __iomem *hdr = ch;
368
369	if (readl(&hdr->cli_state_os) == CHANNELCLI_DISABLED) {
370		if ((readb(&hdr->cli_error_os)
371		     & ULTRA_CLIERROROS_THROTTLEMSG_DISABLED) == 0) {
372			/* we are NOT throttling this message */
373			writeb(readb(&hdr->cli_error_os) |
374			       ULTRA_CLIERROROS_THROTTLEMSG_DISABLED,
375			       &hdr->cli_error_os);
376			/* throttle until acquire successful */
377
378			pr_info("%s Channel StateTransition INVALID! - acquire failed because OS client DISABLED\n",
379				id);
380		}
381		return 0;
382	}
383	if ((readl(&hdr->cli_state_os) != CHANNELCLI_OWNED) &&
384	    (readl(&hdr->cli_state_boot) == CHANNELCLI_DISABLED)) {
385		/* Our competitor is DISABLED, so we can transition to OWNED */
386		pr_info("%s Channel StateTransition (%s) %s(%d)-->%s(%d)\n",
387			id, "cli_state_os",
388			ULTRA_CHANNELCLI_STRING(readl(&hdr->cli_state_os)),
389			readl(&hdr->cli_state_os),
390			ULTRA_CHANNELCLI_STRING(CHANNELCLI_OWNED),
391			CHANNELCLI_OWNED);
392		writel(CHANNELCLI_OWNED, &hdr->cli_state_os);
393		mb(); /* required for channel synch */
394	}
395	if (readl(&hdr->cli_state_os) == CHANNELCLI_OWNED) {
396		if (readb(&hdr->cli_error_os) != 0) {
397			/* we are in an error msg throttling state;
398			 * come out of it */
399			pr_info("%s Channel OS client acquire now successful\n",
400				id);
401			writeb(0, &hdr->cli_error_os);
402		}
403		return 1;
404	}
405
406	/* We have to do it the "hard way".  We transition to BUSY,
407	* and can use the channel iff our competitor has not also
408	* transitioned to BUSY. */
409	if (readl(&hdr->cli_state_os) != CHANNELCLI_ATTACHED) {
410		if ((readb(&hdr->cli_error_os)
411		     & ULTRA_CLIERROROS_THROTTLEMSG_NOTATTACHED) == 0) {
412			/* we are NOT throttling this message */
413			writeb(readb(&hdr->cli_error_os) |
414			       ULTRA_CLIERROROS_THROTTLEMSG_NOTATTACHED,
415			       &hdr->cli_error_os);
416			/* throttle until acquire successful */
417			pr_info("%s Channel StateTransition INVALID! - acquire failed because OS client NOT ATTACHED (state=%s(%d))\n",
418				id, ULTRA_CHANNELCLI_STRING(
419						readl(&hdr->cli_state_os)),
420				readl(&hdr->cli_state_os));
421		}
422		return 0;
423	}
424	writel(CHANNELCLI_BUSY, &hdr->cli_state_os);
425	mb(); /* required for channel synch */
426	if (readl(&hdr->cli_state_boot) == CHANNELCLI_BUSY) {
427		if ((readb(&hdr->cli_error_os)
428		     & ULTRA_CLIERROROS_THROTTLEMSG_BUSY) == 0) {
429			/* we are NOT throttling this message */
430			writeb(readb(&hdr->cli_error_os) |
431			       ULTRA_CLIERROROS_THROTTLEMSG_BUSY,
432			       &hdr->cli_error_os);
433			/* throttle until acquire successful */
434			pr_info("%s Channel StateTransition failed - host OS acquire failed because boot BUSY\n",
435				id);
436		}
437		/* reset busy */
438		writel(CHANNELCLI_ATTACHED, &hdr->cli_state_os);
439		mb(); /* required for channel synch */
440		return 0;
441	}
442	if (readb(&hdr->cli_error_os) != 0) {
443		/* we are in an error msg throttling state; come out of it */
444		pr_info("%s Channel OS client acquire now successful\n", id);
445		writeb(0, &hdr->cli_error_os);
446	}
447	return 1;
448}
449
450static inline void
451spar_channel_client_release_os(void __iomem *ch, u8 *id)
452{
453	struct channel_header __iomem *hdr = ch;
454
455	if (readb(&hdr->cli_error_os) != 0) {
456		/* we are in an error msg throttling state; come out of it */
457		pr_info("%s Channel OS client error state cleared\n", id);
458		writeb(0, &hdr->cli_error_os);
459	}
460	if (readl(&hdr->cli_state_os) == CHANNELCLI_OWNED)
461		return;
462	if (readl(&hdr->cli_state_os) != CHANNELCLI_BUSY) {
463		pr_info("%s Channel StateTransition INVALID! - release failed because OS client NOT BUSY (state=%s(%d))\n",
464			id, ULTRA_CHANNELCLI_STRING(
465					readl(&hdr->cli_state_os)),
466			readl(&hdr->cli_state_os));
467		/* return; */
468	}
469	writel(CHANNELCLI_ATTACHED, &hdr->cli_state_os); /* release busy */
470}
471
472/*
473* Routine Description:
474* Tries to insert the prebuilt signal pointed to by pSignal into the nth
475* Queue of the Channel pointed to by pChannel
476*
477* Parameters:
478* pChannel: (IN) points to the IO Channel
479* Queue: (IN) nth Queue of the IO Channel
480* pSignal: (IN) pointer to the signal
481*
482* Assumptions:
483* - pChannel, Queue and pSignal are valid.
484* - If insertion fails due to a full queue, the caller will determine the
485* retry policy (e.g. wait & try again, report an error, etc.).
486*
487* Return value: 1 if the insertion succeeds, 0 if the queue was
488* full.
489*/
490
491unsigned char spar_signal_insert(struct channel_header __iomem *ch, u32 queue,
492				 void *sig);
493
494/*
495* Routine Description:
496* Removes one signal from Channel pChannel's nth Queue at the
497* time of the call and copies it into the memory pointed to by
498* pSignal.
499*
500* Parameters:
501* pChannel: (IN) points to the IO Channel
502* Queue: (IN) nth Queue of the IO Channel
503* pSignal: (IN) pointer to where the signals are to be copied
504*
505* Assumptions:
506* - pChannel and Queue are valid.
507* - pSignal points to a memory area large enough to hold queue's SignalSize
508*
509* Return value: 1 if the removal succeeds, 0 if the queue was
510* empty.
511*/
512
513unsigned char spar_signal_remove(struct channel_header __iomem *ch, u32 queue,
514				 void *sig);
515
516/*
517* Routine Description:
518* Removes all signals present in Channel pChannel's nth Queue at the
519* time of the call and copies them into the memory pointed to by
520* pSignal.  Returns the # of signals copied as the value of the routine.
521*
522* Parameters:
523* pChannel: (IN) points to the IO Channel
524* Queue: (IN) nth Queue of the IO Channel
525* pSignal: (IN) pointer to where the signals are to be copied
526*
527* Assumptions:
528* - pChannel and Queue are valid.
529* - pSignal points to a memory area large enough to hold Queue's MaxSignals
530* # of signals, each of which is Queue's SignalSize.
531*
532* Return value:
533* # of signals copied.
534*/
535unsigned int spar_signal_remove_all(struct channel_header *ch, u32 queue,
536				    void *sig);
537
538/*
539* Routine Description:
540* Determine whether a signal queue is empty.
541*
542* Parameters:
543* pChannel: (IN) points to the IO Channel
544* Queue: (IN) nth Queue of the IO Channel
545*
546* Return value:
547* 1 if the signal queue is empty, 0 otherwise.
548*/
549unsigned char spar_signalqueue_empty(struct channel_header __iomem *ch,
550				     u32 queue);
551
552#endif
553