1/* sunvnet.c: Sun LDOM Virtual Network Driver. 2 * 3 * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4 */ 5 6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8#include <linux/module.h> 9#include <linux/kernel.h> 10#include <linux/types.h> 11#include <linux/slab.h> 12#include <linux/delay.h> 13#include <linux/init.h> 14#include <linux/netdevice.h> 15#include <linux/ethtool.h> 16#include <linux/etherdevice.h> 17#include <linux/mutex.h> 18#include <linux/highmem.h> 19#include <linux/if_vlan.h> 20 21#if IS_ENABLED(CONFIG_IPV6) 22#include <linux/icmpv6.h> 23#endif 24 25#include <net/ip.h> 26#include <net/icmp.h> 27#include <net/route.h> 28 29#include <asm/vio.h> 30#include <asm/ldc.h> 31 32#include "sunvnet.h" 33 34#define DRV_MODULE_NAME "sunvnet" 35#define DRV_MODULE_VERSION "1.0" 36#define DRV_MODULE_RELDATE "June 25, 2007" 37 38static char version[] = 39 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 40MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); 41MODULE_DESCRIPTION("Sun LDOM virtual network driver"); 42MODULE_LICENSE("GPL"); 43MODULE_VERSION(DRV_MODULE_VERSION); 44 45#define VNET_MAX_TXQS 16 46 47/* Heuristic for the number of times to exponentially backoff and 48 * retry sending an LDC trigger when EAGAIN is encountered 49 */ 50#define VNET_MAX_RETRIES 10 51 52static int __vnet_tx_trigger(struct vnet_port *port, u32 start); 53static void vnet_port_reset(struct vnet_port *port); 54 55/* Ordered from largest major to lowest */ 56static struct vio_version vnet_versions[] = { 57 { .major = 1, .minor = 8 }, 58 { .major = 1, .minor = 7 }, 59 { .major = 1, .minor = 6 }, 60 { .major = 1, .minor = 0 }, 61}; 62 63static inline u32 vnet_tx_dring_avail(struct vio_dring_state *dr) 64{ 65 return vio_dring_avail(dr, VNET_TX_RING_SIZE); 66} 67 68static int vnet_handle_unknown(struct vnet_port *port, void *arg) 69{ 70 struct vio_msg_tag *pkt = arg; 71 72 pr_err("Received unknown msg [%02x:%02x:%04x:%08x]\n", 73 pkt->type, pkt->stype, pkt->stype_env, pkt->sid); 74 pr_err("Resetting connection\n"); 75 76 ldc_disconnect(port->vio.lp); 77 78 return -ECONNRESET; 79} 80 81static int vnet_port_alloc_tx_ring(struct vnet_port *port); 82 83static int vnet_send_attr(struct vio_driver_state *vio) 84{ 85 struct vnet_port *port = to_vnet_port(vio); 86 struct net_device *dev = port->vp->dev; 87 struct vio_net_attr_info pkt; 88 int framelen = ETH_FRAME_LEN; 89 int i, err; 90 91 err = vnet_port_alloc_tx_ring(to_vnet_port(vio)); 92 if (err) 93 return err; 94 95 memset(&pkt, 0, sizeof(pkt)); 96 pkt.tag.type = VIO_TYPE_CTRL; 97 pkt.tag.stype = VIO_SUBTYPE_INFO; 98 pkt.tag.stype_env = VIO_ATTR_INFO; 99 pkt.tag.sid = vio_send_sid(vio); 100 if (vio_version_before(vio, 1, 2)) 101 pkt.xfer_mode = VIO_DRING_MODE; 102 else 103 pkt.xfer_mode = VIO_NEW_DRING_MODE; 104 pkt.addr_type = VNET_ADDR_ETHERMAC; 105 pkt.ack_freq = 0; 106 for (i = 0; i < 6; i++) 107 pkt.addr |= (u64)dev->dev_addr[i] << ((5 - i) * 8); 108 if (vio_version_after(vio, 1, 3)) { 109 if (port->rmtu) { 110 port->rmtu = min(VNET_MAXPACKET, port->rmtu); 111 pkt.mtu = port->rmtu; 112 } else { 113 port->rmtu = VNET_MAXPACKET; 114 pkt.mtu = port->rmtu; 115 } 116 if (vio_version_after_eq(vio, 1, 6)) 117 pkt.options = VIO_TX_DRING; 118 } else if (vio_version_before(vio, 1, 3)) { 119 pkt.mtu = framelen; 120 } else { /* v1.3 */ 121 pkt.mtu = framelen + VLAN_HLEN; 122 } 123 124 pkt.cflags = 0; 125 if (vio_version_after_eq(vio, 1, 7) && port->tso) { 126 pkt.cflags |= VNET_LSO_IPV4_CAPAB; 127 if (!port->tsolen) 128 port->tsolen = VNET_MAXTSO; 129 pkt.ipv4_lso_maxlen = port->tsolen; 130 } 131 132 pkt.plnk_updt = PHYSLINK_UPDATE_NONE; 133 134 viodbg(HS, "SEND NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 135 "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 136 "cflags[0x%04x] lso_max[%u]\n", 137 pkt.xfer_mode, pkt.addr_type, 138 (unsigned long long)pkt.addr, 139 pkt.ack_freq, pkt.plnk_updt, pkt.options, 140 (unsigned long long)pkt.mtu, pkt.cflags, pkt.ipv4_lso_maxlen); 141 142 143 return vio_ldc_send(vio, &pkt, sizeof(pkt)); 144} 145 146static int handle_attr_info(struct vio_driver_state *vio, 147 struct vio_net_attr_info *pkt) 148{ 149 struct vnet_port *port = to_vnet_port(vio); 150 u64 localmtu; 151 u8 xfer_mode; 152 153 viodbg(HS, "GOT NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 154 "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 155 " (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 156 pkt->xfer_mode, pkt->addr_type, 157 (unsigned long long)pkt->addr, 158 pkt->ack_freq, pkt->plnk_updt, pkt->options, 159 (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 160 pkt->ipv4_lso_maxlen); 161 162 pkt->tag.sid = vio_send_sid(vio); 163 164 xfer_mode = pkt->xfer_mode; 165 /* for version < 1.2, VIO_DRING_MODE = 0x3 and no bitmask */ 166 if (vio_version_before(vio, 1, 2) && xfer_mode == VIO_DRING_MODE) 167 xfer_mode = VIO_NEW_DRING_MODE; 168 169 /* MTU negotiation: 170 * < v1.3 - ETH_FRAME_LEN exactly 171 * > v1.3 - MIN(pkt.mtu, VNET_MAXPACKET, port->rmtu) and change 172 * pkt->mtu for ACK 173 * = v1.3 - ETH_FRAME_LEN + VLAN_HLEN exactly 174 */ 175 if (vio_version_before(vio, 1, 3)) { 176 localmtu = ETH_FRAME_LEN; 177 } else if (vio_version_after(vio, 1, 3)) { 178 localmtu = port->rmtu ? port->rmtu : VNET_MAXPACKET; 179 localmtu = min(pkt->mtu, localmtu); 180 pkt->mtu = localmtu; 181 } else { /* v1.3 */ 182 localmtu = ETH_FRAME_LEN + VLAN_HLEN; 183 } 184 port->rmtu = localmtu; 185 186 /* LSO negotiation */ 187 if (vio_version_after_eq(vio, 1, 7)) 188 port->tso &= !!(pkt->cflags & VNET_LSO_IPV4_CAPAB); 189 else 190 port->tso = false; 191 if (port->tso) { 192 if (!port->tsolen) 193 port->tsolen = VNET_MAXTSO; 194 port->tsolen = min(port->tsolen, pkt->ipv4_lso_maxlen); 195 if (port->tsolen < VNET_MINTSO) { 196 port->tso = false; 197 port->tsolen = 0; 198 pkt->cflags &= ~VNET_LSO_IPV4_CAPAB; 199 } 200 pkt->ipv4_lso_maxlen = port->tsolen; 201 } else { 202 pkt->cflags &= ~VNET_LSO_IPV4_CAPAB; 203 pkt->ipv4_lso_maxlen = 0; 204 } 205 206 /* for version >= 1.6, ACK packet mode we support */ 207 if (vio_version_after_eq(vio, 1, 6)) { 208 pkt->xfer_mode = VIO_NEW_DRING_MODE; 209 pkt->options = VIO_TX_DRING; 210 } 211 212 if (!(xfer_mode | VIO_NEW_DRING_MODE) || 213 pkt->addr_type != VNET_ADDR_ETHERMAC || 214 pkt->mtu != localmtu) { 215 viodbg(HS, "SEND NET ATTR NACK\n"); 216 217 pkt->tag.stype = VIO_SUBTYPE_NACK; 218 219 (void) vio_ldc_send(vio, pkt, sizeof(*pkt)); 220 221 return -ECONNRESET; 222 } else { 223 viodbg(HS, "SEND NET ATTR ACK xmode[0x%x] atype[0x%x] " 224 "addr[%llx] ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] " 225 "mtu[%llu] (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 226 pkt->xfer_mode, pkt->addr_type, 227 (unsigned long long)pkt->addr, 228 pkt->ack_freq, pkt->plnk_updt, pkt->options, 229 (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 230 pkt->ipv4_lso_maxlen); 231 232 pkt->tag.stype = VIO_SUBTYPE_ACK; 233 234 return vio_ldc_send(vio, pkt, sizeof(*pkt)); 235 } 236 237} 238 239static int handle_attr_ack(struct vio_driver_state *vio, 240 struct vio_net_attr_info *pkt) 241{ 242 viodbg(HS, "GOT NET ATTR ACK\n"); 243 244 return 0; 245} 246 247static int handle_attr_nack(struct vio_driver_state *vio, 248 struct vio_net_attr_info *pkt) 249{ 250 viodbg(HS, "GOT NET ATTR NACK\n"); 251 252 return -ECONNRESET; 253} 254 255static int vnet_handle_attr(struct vio_driver_state *vio, void *arg) 256{ 257 struct vio_net_attr_info *pkt = arg; 258 259 switch (pkt->tag.stype) { 260 case VIO_SUBTYPE_INFO: 261 return handle_attr_info(vio, pkt); 262 263 case VIO_SUBTYPE_ACK: 264 return handle_attr_ack(vio, pkt); 265 266 case VIO_SUBTYPE_NACK: 267 return handle_attr_nack(vio, pkt); 268 269 default: 270 return -ECONNRESET; 271 } 272} 273 274static void vnet_handshake_complete(struct vio_driver_state *vio) 275{ 276 struct vio_dring_state *dr; 277 278 dr = &vio->drings[VIO_DRIVER_RX_RING]; 279 dr->snd_nxt = dr->rcv_nxt = 1; 280 281 dr = &vio->drings[VIO_DRIVER_TX_RING]; 282 dr->snd_nxt = dr->rcv_nxt = 1; 283} 284 285/* The hypervisor interface that implements copying to/from imported 286 * memory from another domain requires that copies are done to 8-byte 287 * aligned buffers, and that the lengths of such copies are also 8-byte 288 * multiples. 289 * 290 * So we align skb->data to an 8-byte multiple and pad-out the data 291 * area so we can round the copy length up to the next multiple of 292 * 8 for the copy. 293 * 294 * The transmitter puts the actual start of the packet 6 bytes into 295 * the buffer it sends over, so that the IP headers after the ethernet 296 * header are aligned properly. These 6 bytes are not in the descriptor 297 * length, they are simply implied. This offset is represented using 298 * the VNET_PACKET_SKIP macro. 299 */ 300static struct sk_buff *alloc_and_align_skb(struct net_device *dev, 301 unsigned int len) 302{ 303 struct sk_buff *skb = netdev_alloc_skb(dev, len+VNET_PACKET_SKIP+8+8); 304 unsigned long addr, off; 305 306 if (unlikely(!skb)) 307 return NULL; 308 309 addr = (unsigned long) skb->data; 310 off = ((addr + 7UL) & ~7UL) - addr; 311 if (off) 312 skb_reserve(skb, off); 313 314 return skb; 315} 316 317static inline void vnet_fullcsum(struct sk_buff *skb) 318{ 319 struct iphdr *iph = ip_hdr(skb); 320 int offset = skb_transport_offset(skb); 321 322 if (skb->protocol != htons(ETH_P_IP)) 323 return; 324 if (iph->protocol != IPPROTO_TCP && 325 iph->protocol != IPPROTO_UDP) 326 return; 327 skb->ip_summed = CHECKSUM_NONE; 328 skb->csum_level = 1; 329 skb->csum = 0; 330 if (iph->protocol == IPPROTO_TCP) { 331 struct tcphdr *ptcp = tcp_hdr(skb); 332 333 ptcp->check = 0; 334 skb->csum = skb_checksum(skb, offset, skb->len - offset, 0); 335 ptcp->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 336 skb->len - offset, IPPROTO_TCP, 337 skb->csum); 338 } else if (iph->protocol == IPPROTO_UDP) { 339 struct udphdr *pudp = udp_hdr(skb); 340 341 pudp->check = 0; 342 skb->csum = skb_checksum(skb, offset, skb->len - offset, 0); 343 pudp->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 344 skb->len - offset, IPPROTO_UDP, 345 skb->csum); 346 } 347} 348 349static int vnet_rx_one(struct vnet_port *port, struct vio_net_desc *desc) 350{ 351 struct net_device *dev = port->vp->dev; 352 unsigned int len = desc->size; 353 unsigned int copy_len; 354 struct sk_buff *skb; 355 int maxlen; 356 int err; 357 358 err = -EMSGSIZE; 359 if (port->tso && port->tsolen > port->rmtu) 360 maxlen = port->tsolen; 361 else 362 maxlen = port->rmtu; 363 if (unlikely(len < ETH_ZLEN || len > maxlen)) { 364 dev->stats.rx_length_errors++; 365 goto out_dropped; 366 } 367 368 skb = alloc_and_align_skb(dev, len); 369 err = -ENOMEM; 370 if (unlikely(!skb)) { 371 dev->stats.rx_missed_errors++; 372 goto out_dropped; 373 } 374 375 copy_len = (len + VNET_PACKET_SKIP + 7U) & ~7U; 376 skb_put(skb, copy_len); 377 err = ldc_copy(port->vio.lp, LDC_COPY_IN, 378 skb->data, copy_len, 0, 379 desc->cookies, desc->ncookies); 380 if (unlikely(err < 0)) { 381 dev->stats.rx_frame_errors++; 382 goto out_free_skb; 383 } 384 385 skb_pull(skb, VNET_PACKET_SKIP); 386 skb_trim(skb, len); 387 skb->protocol = eth_type_trans(skb, dev); 388 389 if (vio_version_after_eq(&port->vio, 1, 8)) { 390 struct vio_net_dext *dext = vio_net_ext(desc); 391 392 if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM) { 393 if (skb->protocol == ETH_P_IP) { 394 struct iphdr *iph = (struct iphdr *)skb->data; 395 396 iph->check = 0; 397 ip_send_check(iph); 398 } 399 } 400 if ((dext->flags & VNET_PKT_HCK_FULLCKSUM) && 401 skb->ip_summed == CHECKSUM_NONE) 402 vnet_fullcsum(skb); 403 if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM_OK) { 404 skb->ip_summed = CHECKSUM_PARTIAL; 405 skb->csum_level = 0; 406 if (dext->flags & VNET_PKT_HCK_FULLCKSUM_OK) 407 skb->csum_level = 1; 408 } 409 } 410 411 skb->ip_summed = port->switch_port ? CHECKSUM_NONE : CHECKSUM_PARTIAL; 412 413 dev->stats.rx_packets++; 414 dev->stats.rx_bytes += len; 415 napi_gro_receive(&port->napi, skb); 416 return 0; 417 418out_free_skb: 419 kfree_skb(skb); 420 421out_dropped: 422 dev->stats.rx_dropped++; 423 return err; 424} 425 426static int vnet_send_ack(struct vnet_port *port, struct vio_dring_state *dr, 427 u32 start, u32 end, u8 vio_dring_state) 428{ 429 struct vio_dring_data hdr = { 430 .tag = { 431 .type = VIO_TYPE_DATA, 432 .stype = VIO_SUBTYPE_ACK, 433 .stype_env = VIO_DRING_DATA, 434 .sid = vio_send_sid(&port->vio), 435 }, 436 .dring_ident = dr->ident, 437 .start_idx = start, 438 .end_idx = end, 439 .state = vio_dring_state, 440 }; 441 int err, delay; 442 int retries = 0; 443 444 hdr.seq = dr->snd_nxt; 445 delay = 1; 446 do { 447 err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 448 if (err > 0) { 449 dr->snd_nxt++; 450 break; 451 } 452 udelay(delay); 453 if ((delay <<= 1) > 128) 454 delay = 128; 455 if (retries++ > VNET_MAX_RETRIES) { 456 pr_info("ECONNRESET %x:%x:%x:%x:%x:%x\n", 457 port->raddr[0], port->raddr[1], 458 port->raddr[2], port->raddr[3], 459 port->raddr[4], port->raddr[5]); 460 break; 461 } 462 } while (err == -EAGAIN); 463 464 if (err <= 0 && vio_dring_state == VIO_DRING_STOPPED) { 465 port->stop_rx_idx = end; 466 port->stop_rx = true; 467 } else { 468 port->stop_rx_idx = 0; 469 port->stop_rx = false; 470 } 471 472 return err; 473} 474 475static struct vio_net_desc *get_rx_desc(struct vnet_port *port, 476 struct vio_dring_state *dr, 477 u32 index) 478{ 479 struct vio_net_desc *desc = port->vio.desc_buf; 480 int err; 481 482 err = ldc_get_dring_entry(port->vio.lp, desc, dr->entry_size, 483 (index * dr->entry_size), 484 dr->cookies, dr->ncookies); 485 if (err < 0) 486 return ERR_PTR(err); 487 488 return desc; 489} 490 491static int put_rx_desc(struct vnet_port *port, 492 struct vio_dring_state *dr, 493 struct vio_net_desc *desc, 494 u32 index) 495{ 496 int err; 497 498 err = ldc_put_dring_entry(port->vio.lp, desc, dr->entry_size, 499 (index * dr->entry_size), 500 dr->cookies, dr->ncookies); 501 if (err < 0) 502 return err; 503 504 return 0; 505} 506 507static int vnet_walk_rx_one(struct vnet_port *port, 508 struct vio_dring_state *dr, 509 u32 index, int *needs_ack) 510{ 511 struct vio_net_desc *desc = get_rx_desc(port, dr, index); 512 struct vio_driver_state *vio = &port->vio; 513 int err; 514 515 BUG_ON(desc == NULL); 516 if (IS_ERR(desc)) 517 return PTR_ERR(desc); 518 519 if (desc->hdr.state != VIO_DESC_READY) 520 return 1; 521 522 dma_rmb(); 523 524 viodbg(DATA, "vio_walk_rx_one desc[%02x:%02x:%08x:%08x:%llx:%llx]\n", 525 desc->hdr.state, desc->hdr.ack, 526 desc->size, desc->ncookies, 527 desc->cookies[0].cookie_addr, 528 desc->cookies[0].cookie_size); 529 530 err = vnet_rx_one(port, desc); 531 if (err == -ECONNRESET) 532 return err; 533 desc->hdr.state = VIO_DESC_DONE; 534 err = put_rx_desc(port, dr, desc, index); 535 if (err < 0) 536 return err; 537 *needs_ack = desc->hdr.ack; 538 return 0; 539} 540 541static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr, 542 u32 start, u32 end, int *npkts, int budget) 543{ 544 struct vio_driver_state *vio = &port->vio; 545 int ack_start = -1, ack_end = -1; 546 bool send_ack = true; 547 548 end = (end == (u32) -1) ? vio_dring_prev(dr, start) 549 : vio_dring_next(dr, end); 550 551 viodbg(DATA, "vnet_walk_rx start[%08x] end[%08x]\n", start, end); 552 553 while (start != end) { 554 int ack = 0, err = vnet_walk_rx_one(port, dr, start, &ack); 555 if (err == -ECONNRESET) 556 return err; 557 if (err != 0) 558 break; 559 (*npkts)++; 560 if (ack_start == -1) 561 ack_start = start; 562 ack_end = start; 563 start = vio_dring_next(dr, start); 564 if (ack && start != end) { 565 err = vnet_send_ack(port, dr, ack_start, ack_end, 566 VIO_DRING_ACTIVE); 567 if (err == -ECONNRESET) 568 return err; 569 ack_start = -1; 570 } 571 if ((*npkts) >= budget) { 572 send_ack = false; 573 break; 574 } 575 } 576 if (unlikely(ack_start == -1)) 577 ack_start = ack_end = vio_dring_prev(dr, start); 578 if (send_ack) { 579 port->napi_resume = false; 580 return vnet_send_ack(port, dr, ack_start, ack_end, 581 VIO_DRING_STOPPED); 582 } else { 583 port->napi_resume = true; 584 port->napi_stop_idx = ack_end; 585 return 1; 586 } 587} 588 589static int vnet_rx(struct vnet_port *port, void *msgbuf, int *npkts, 590 int budget) 591{ 592 struct vio_dring_data *pkt = msgbuf; 593 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_RX_RING]; 594 struct vio_driver_state *vio = &port->vio; 595 596 viodbg(DATA, "vnet_rx stype_env[%04x] seq[%016llx] rcv_nxt[%016llx]\n", 597 pkt->tag.stype_env, pkt->seq, dr->rcv_nxt); 598 599 if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 600 return 0; 601 if (unlikely(pkt->seq != dr->rcv_nxt)) { 602 pr_err("RX out of sequence seq[0x%llx] rcv_nxt[0x%llx]\n", 603 pkt->seq, dr->rcv_nxt); 604 return 0; 605 } 606 607 if (!port->napi_resume) 608 dr->rcv_nxt++; 609 610 /* XXX Validate pkt->start_idx and pkt->end_idx XXX */ 611 612 return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx, 613 npkts, budget); 614} 615 616static int idx_is_pending(struct vio_dring_state *dr, u32 end) 617{ 618 u32 idx = dr->cons; 619 int found = 0; 620 621 while (idx != dr->prod) { 622 if (idx == end) { 623 found = 1; 624 break; 625 } 626 idx = vio_dring_next(dr, idx); 627 } 628 return found; 629} 630 631static int vnet_ack(struct vnet_port *port, void *msgbuf) 632{ 633 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 634 struct vio_dring_data *pkt = msgbuf; 635 struct net_device *dev; 636 struct vnet *vp; 637 u32 end; 638 struct vio_net_desc *desc; 639 struct netdev_queue *txq; 640 641 if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 642 return 0; 643 644 end = pkt->end_idx; 645 vp = port->vp; 646 dev = vp->dev; 647 netif_tx_lock(dev); 648 if (unlikely(!idx_is_pending(dr, end))) { 649 netif_tx_unlock(dev); 650 return 0; 651 } 652 653 /* sync for race conditions with vnet_start_xmit() and tell xmit it 654 * is time to send a trigger. 655 */ 656 dr->cons = vio_dring_next(dr, end); 657 desc = vio_dring_entry(dr, dr->cons); 658 if (desc->hdr.state == VIO_DESC_READY && !port->start_cons) { 659 /* vnet_start_xmit() just populated this dring but missed 660 * sending the "start" LDC message to the consumer. 661 * Send a "start" trigger on its behalf. 662 */ 663 if (__vnet_tx_trigger(port, dr->cons) > 0) 664 port->start_cons = false; 665 else 666 port->start_cons = true; 667 } else { 668 port->start_cons = true; 669 } 670 netif_tx_unlock(dev); 671 672 txq = netdev_get_tx_queue(dev, port->q_index); 673 if (unlikely(netif_tx_queue_stopped(txq) && 674 vnet_tx_dring_avail(dr) >= VNET_TX_WAKEUP_THRESH(dr))) 675 return 1; 676 677 return 0; 678} 679 680static int vnet_nack(struct vnet_port *port, void *msgbuf) 681{ 682 /* XXX just reset or similar XXX */ 683 return 0; 684} 685 686static int handle_mcast(struct vnet_port *port, void *msgbuf) 687{ 688 struct vio_net_mcast_info *pkt = msgbuf; 689 690 if (pkt->tag.stype != VIO_SUBTYPE_ACK) 691 pr_err("%s: Got unexpected MCAST reply [%02x:%02x:%04x:%08x]\n", 692 port->vp->dev->name, 693 pkt->tag.type, 694 pkt->tag.stype, 695 pkt->tag.stype_env, 696 pkt->tag.sid); 697 698 return 0; 699} 700 701/* Got back a STOPPED LDC message on port. If the queue is stopped, 702 * wake it up so that we'll send out another START message at the 703 * next TX. 704 */ 705static void maybe_tx_wakeup(struct vnet_port *port) 706{ 707 struct netdev_queue *txq; 708 709 txq = netdev_get_tx_queue(port->vp->dev, port->q_index); 710 __netif_tx_lock(txq, smp_processor_id()); 711 if (likely(netif_tx_queue_stopped(txq))) { 712 struct vio_dring_state *dr; 713 714 dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 715 netif_tx_wake_queue(txq); 716 } 717 __netif_tx_unlock(txq); 718} 719 720static inline bool port_is_up(struct vnet_port *vnet) 721{ 722 struct vio_driver_state *vio = &vnet->vio; 723 724 return !!(vio->hs_state & VIO_HS_COMPLETE); 725} 726 727static int vnet_event_napi(struct vnet_port *port, int budget) 728{ 729 struct vio_driver_state *vio = &port->vio; 730 int tx_wakeup, err; 731 int npkts = 0; 732 int event = (port->rx_event & LDC_EVENT_RESET); 733 734ldc_ctrl: 735 if (unlikely(event == LDC_EVENT_RESET || 736 event == LDC_EVENT_UP)) { 737 vio_link_state_change(vio, event); 738 739 if (event == LDC_EVENT_RESET) { 740 vnet_port_reset(port); 741 vio_port_up(vio); 742 } 743 port->rx_event = 0; 744 return 0; 745 } 746 /* We may have multiple LDC events in rx_event. Unroll send_events() */ 747 event = (port->rx_event & LDC_EVENT_UP); 748 port->rx_event &= ~(LDC_EVENT_RESET|LDC_EVENT_UP); 749 if (event == LDC_EVENT_UP) 750 goto ldc_ctrl; 751 event = port->rx_event; 752 if (!(event & LDC_EVENT_DATA_READY)) 753 return 0; 754 755 /* we dont expect any other bits than RESET, UP, DATA_READY */ 756 BUG_ON(event != LDC_EVENT_DATA_READY); 757 758 tx_wakeup = err = 0; 759 while (1) { 760 union { 761 struct vio_msg_tag tag; 762 u64 raw[8]; 763 } msgbuf; 764 765 if (port->napi_resume) { 766 struct vio_dring_data *pkt = 767 (struct vio_dring_data *)&msgbuf; 768 struct vio_dring_state *dr = 769 &port->vio.drings[VIO_DRIVER_RX_RING]; 770 771 pkt->tag.type = VIO_TYPE_DATA; 772 pkt->tag.stype = VIO_SUBTYPE_INFO; 773 pkt->tag.stype_env = VIO_DRING_DATA; 774 pkt->seq = dr->rcv_nxt; 775 pkt->start_idx = vio_dring_next(dr, port->napi_stop_idx); 776 pkt->end_idx = -1; 777 goto napi_resume; 778 } 779 err = ldc_read(vio->lp, &msgbuf, sizeof(msgbuf)); 780 if (unlikely(err < 0)) { 781 if (err == -ECONNRESET) 782 vio_conn_reset(vio); 783 break; 784 } 785 if (err == 0) 786 break; 787 viodbg(DATA, "TAG [%02x:%02x:%04x:%08x]\n", 788 msgbuf.tag.type, 789 msgbuf.tag.stype, 790 msgbuf.tag.stype_env, 791 msgbuf.tag.sid); 792 err = vio_validate_sid(vio, &msgbuf.tag); 793 if (err < 0) 794 break; 795napi_resume: 796 if (likely(msgbuf.tag.type == VIO_TYPE_DATA)) { 797 if (msgbuf.tag.stype == VIO_SUBTYPE_INFO) { 798 if (!port_is_up(port)) { 799 /* failures like handshake_failure() 800 * may have cleaned up dring, but 801 * NAPI polling may bring us here. 802 */ 803 err = -ECONNRESET; 804 break; 805 } 806 err = vnet_rx(port, &msgbuf, &npkts, budget); 807 if (npkts >= budget) 808 break; 809 if (npkts == 0) 810 break; 811 } else if (msgbuf.tag.stype == VIO_SUBTYPE_ACK) { 812 err = vnet_ack(port, &msgbuf); 813 if (err > 0) 814 tx_wakeup |= err; 815 } else if (msgbuf.tag.stype == VIO_SUBTYPE_NACK) { 816 err = vnet_nack(port, &msgbuf); 817 } 818 } else if (msgbuf.tag.type == VIO_TYPE_CTRL) { 819 if (msgbuf.tag.stype_env == VNET_MCAST_INFO) 820 err = handle_mcast(port, &msgbuf); 821 else 822 err = vio_control_pkt_engine(vio, &msgbuf); 823 if (err) 824 break; 825 } else { 826 err = vnet_handle_unknown(port, &msgbuf); 827 } 828 if (err == -ECONNRESET) 829 break; 830 } 831 if (unlikely(tx_wakeup && err != -ECONNRESET)) 832 maybe_tx_wakeup(port); 833 return npkts; 834} 835 836static int vnet_poll(struct napi_struct *napi, int budget) 837{ 838 struct vnet_port *port = container_of(napi, struct vnet_port, napi); 839 struct vio_driver_state *vio = &port->vio; 840 int processed = vnet_event_napi(port, budget); 841 842 if (processed < budget) { 843 napi_complete(napi); 844 port->rx_event &= ~LDC_EVENT_DATA_READY; 845 vio_set_intr(vio->vdev->rx_ino, HV_INTR_ENABLED); 846 } 847 return processed; 848} 849 850static void vnet_event(void *arg, int event) 851{ 852 struct vnet_port *port = arg; 853 struct vio_driver_state *vio = &port->vio; 854 855 port->rx_event |= event; 856 vio_set_intr(vio->vdev->rx_ino, HV_INTR_DISABLED); 857 napi_schedule(&port->napi); 858 859} 860 861static int __vnet_tx_trigger(struct vnet_port *port, u32 start) 862{ 863 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 864 struct vio_dring_data hdr = { 865 .tag = { 866 .type = VIO_TYPE_DATA, 867 .stype = VIO_SUBTYPE_INFO, 868 .stype_env = VIO_DRING_DATA, 869 .sid = vio_send_sid(&port->vio), 870 }, 871 .dring_ident = dr->ident, 872 .start_idx = start, 873 .end_idx = (u32) -1, 874 }; 875 int err, delay; 876 int retries = 0; 877 878 if (port->stop_rx) { 879 err = vnet_send_ack(port, 880 &port->vio.drings[VIO_DRIVER_RX_RING], 881 port->stop_rx_idx, -1, 882 VIO_DRING_STOPPED); 883 if (err <= 0) 884 return err; 885 } 886 887 hdr.seq = dr->snd_nxt; 888 delay = 1; 889 do { 890 err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 891 if (err > 0) { 892 dr->snd_nxt++; 893 break; 894 } 895 udelay(delay); 896 if ((delay <<= 1) > 128) 897 delay = 128; 898 if (retries++ > VNET_MAX_RETRIES) 899 break; 900 } while (err == -EAGAIN); 901 902 return err; 903} 904 905struct vnet_port *__tx_port_find(struct vnet *vp, struct sk_buff *skb) 906{ 907 unsigned int hash = vnet_hashfn(skb->data); 908 struct hlist_head *hp = &vp->port_hash[hash]; 909 struct vnet_port *port; 910 911 hlist_for_each_entry_rcu(port, hp, hash) { 912 if (!port_is_up(port)) 913 continue; 914 if (ether_addr_equal(port->raddr, skb->data)) 915 return port; 916 } 917 list_for_each_entry_rcu(port, &vp->port_list, list) { 918 if (!port->switch_port) 919 continue; 920 if (!port_is_up(port)) 921 continue; 922 return port; 923 } 924 return NULL; 925} 926 927static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port, 928 unsigned *pending) 929{ 930 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 931 struct sk_buff *skb = NULL; 932 int i, txi; 933 934 *pending = 0; 935 936 txi = dr->prod; 937 for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 938 struct vio_net_desc *d; 939 940 --txi; 941 if (txi < 0) 942 txi = VNET_TX_RING_SIZE-1; 943 944 d = vio_dring_entry(dr, txi); 945 946 if (d->hdr.state == VIO_DESC_READY) { 947 (*pending)++; 948 continue; 949 } 950 if (port->tx_bufs[txi].skb) { 951 if (d->hdr.state != VIO_DESC_DONE) 952 pr_notice("invalid ring buffer state %d\n", 953 d->hdr.state); 954 BUG_ON(port->tx_bufs[txi].skb->next); 955 956 port->tx_bufs[txi].skb->next = skb; 957 skb = port->tx_bufs[txi].skb; 958 port->tx_bufs[txi].skb = NULL; 959 960 ldc_unmap(port->vio.lp, 961 port->tx_bufs[txi].cookies, 962 port->tx_bufs[txi].ncookies); 963 } else if (d->hdr.state == VIO_DESC_FREE) 964 break; 965 d->hdr.state = VIO_DESC_FREE; 966 } 967 return skb; 968} 969 970static inline void vnet_free_skbs(struct sk_buff *skb) 971{ 972 struct sk_buff *next; 973 974 while (skb) { 975 next = skb->next; 976 skb->next = NULL; 977 dev_kfree_skb(skb); 978 skb = next; 979 } 980} 981 982static void vnet_clean_timer_expire(unsigned long port0) 983{ 984 struct vnet_port *port = (struct vnet_port *)port0; 985 struct sk_buff *freeskbs; 986 unsigned pending; 987 988 netif_tx_lock(port->vp->dev); 989 freeskbs = vnet_clean_tx_ring(port, &pending); 990 netif_tx_unlock(port->vp->dev); 991 992 vnet_free_skbs(freeskbs); 993 994 if (pending) 995 (void)mod_timer(&port->clean_timer, 996 jiffies + VNET_CLEAN_TIMEOUT); 997 else 998 del_timer(&port->clean_timer); 999} 1000 1001static inline int vnet_skb_map(struct ldc_channel *lp, struct sk_buff *skb, 1002 struct ldc_trans_cookie *cookies, int ncookies, 1003 unsigned int map_perm) 1004{ 1005 int i, nc, err, blen; 1006 1007 /* header */ 1008 blen = skb_headlen(skb); 1009 if (blen < ETH_ZLEN) 1010 blen = ETH_ZLEN; 1011 blen += VNET_PACKET_SKIP; 1012 blen += 8 - (blen & 7); 1013 1014 err = ldc_map_single(lp, skb->data-VNET_PACKET_SKIP, blen, cookies, 1015 ncookies, map_perm); 1016 if (err < 0) 1017 return err; 1018 nc = err; 1019 1020 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1021 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1022 u8 *vaddr; 1023 1024 if (nc < ncookies) { 1025 vaddr = kmap_atomic(skb_frag_page(f)); 1026 blen = skb_frag_size(f); 1027 blen += 8 - (blen & 7); 1028 err = ldc_map_single(lp, vaddr + f->page_offset, 1029 blen, cookies + nc, ncookies - nc, 1030 map_perm); 1031 kunmap_atomic(vaddr); 1032 } else { 1033 err = -EMSGSIZE; 1034 } 1035 1036 if (err < 0) { 1037 ldc_unmap(lp, cookies, nc); 1038 return err; 1039 } 1040 nc += err; 1041 } 1042 return nc; 1043} 1044 1045static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, int ncookies) 1046{ 1047 struct sk_buff *nskb; 1048 int i, len, pad, docopy; 1049 1050 len = skb->len; 1051 pad = 0; 1052 if (len < ETH_ZLEN) { 1053 pad += ETH_ZLEN - skb->len; 1054 len += pad; 1055 } 1056 len += VNET_PACKET_SKIP; 1057 pad += 8 - (len & 7); 1058 1059 /* make sure we have enough cookies and alignment in every frag */ 1060 docopy = skb_shinfo(skb)->nr_frags >= ncookies; 1061 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1062 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1063 1064 docopy |= f->page_offset & 7; 1065 } 1066 if (((unsigned long)skb->data & 7) != VNET_PACKET_SKIP || 1067 skb_tailroom(skb) < pad || 1068 skb_headroom(skb) < VNET_PACKET_SKIP || docopy) { 1069 int start = 0, offset; 1070 __wsum csum; 1071 1072 len = skb->len > ETH_ZLEN ? skb->len : ETH_ZLEN; 1073 nskb = alloc_and_align_skb(skb->dev, len); 1074 if (nskb == NULL) { 1075 dev_kfree_skb(skb); 1076 return NULL; 1077 } 1078 skb_reserve(nskb, VNET_PACKET_SKIP); 1079 1080 nskb->protocol = skb->protocol; 1081 offset = skb_mac_header(skb) - skb->data; 1082 skb_set_mac_header(nskb, offset); 1083 offset = skb_network_header(skb) - skb->data; 1084 skb_set_network_header(nskb, offset); 1085 offset = skb_transport_header(skb) - skb->data; 1086 skb_set_transport_header(nskb, offset); 1087 1088 offset = 0; 1089 nskb->csum_offset = skb->csum_offset; 1090 nskb->ip_summed = skb->ip_summed; 1091 1092 if (skb->ip_summed == CHECKSUM_PARTIAL) 1093 start = skb_checksum_start_offset(skb); 1094 if (start) { 1095 struct iphdr *iph = ip_hdr(nskb); 1096 int offset = start + nskb->csum_offset; 1097 1098 if (skb_copy_bits(skb, 0, nskb->data, start)) { 1099 dev_kfree_skb(nskb); 1100 dev_kfree_skb(skb); 1101 return NULL; 1102 } 1103 *(__sum16 *)(skb->data + offset) = 0; 1104 csum = skb_copy_and_csum_bits(skb, start, 1105 nskb->data + start, 1106 skb->len - start, 0); 1107 if (iph->protocol == IPPROTO_TCP || 1108 iph->protocol == IPPROTO_UDP) { 1109 csum = csum_tcpudp_magic(iph->saddr, iph->daddr, 1110 skb->len - start, 1111 iph->protocol, csum); 1112 } 1113 *(__sum16 *)(nskb->data + offset) = csum; 1114 1115 nskb->ip_summed = CHECKSUM_NONE; 1116 } else if (skb_copy_bits(skb, 0, nskb->data, skb->len)) { 1117 dev_kfree_skb(nskb); 1118 dev_kfree_skb(skb); 1119 return NULL; 1120 } 1121 (void)skb_put(nskb, skb->len); 1122 if (skb_is_gso(skb)) { 1123 skb_shinfo(nskb)->gso_size = skb_shinfo(skb)->gso_size; 1124 skb_shinfo(nskb)->gso_type = skb_shinfo(skb)->gso_type; 1125 } 1126 nskb->queue_mapping = skb->queue_mapping; 1127 dev_kfree_skb(skb); 1128 skb = nskb; 1129 } 1130 return skb; 1131} 1132 1133static u16 1134vnet_select_queue(struct net_device *dev, struct sk_buff *skb, 1135 void *accel_priv, select_queue_fallback_t fallback) 1136{ 1137 struct vnet *vp = netdev_priv(dev); 1138 struct vnet_port *port = __tx_port_find(vp, skb); 1139 1140 if (port == NULL) 1141 return 0; 1142 return port->q_index; 1143} 1144 1145static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev); 1146 1147static int vnet_handle_offloads(struct vnet_port *port, struct sk_buff *skb) 1148{ 1149 struct net_device *dev = port->vp->dev; 1150 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1151 struct sk_buff *segs; 1152 int maclen, datalen; 1153 int status; 1154 int gso_size, gso_type, gso_segs; 1155 int hlen = skb_transport_header(skb) - skb_mac_header(skb); 1156 int proto = IPPROTO_IP; 1157 1158 if (skb->protocol == htons(ETH_P_IP)) 1159 proto = ip_hdr(skb)->protocol; 1160 else if (skb->protocol == htons(ETH_P_IPV6)) 1161 proto = ipv6_hdr(skb)->nexthdr; 1162 1163 if (proto == IPPROTO_TCP) 1164 hlen += tcp_hdr(skb)->doff * 4; 1165 else if (proto == IPPROTO_UDP) 1166 hlen += sizeof(struct udphdr); 1167 else { 1168 pr_err("vnet_handle_offloads GSO with unknown transport " 1169 "protocol %d tproto %d\n", skb->protocol, proto); 1170 hlen = 128; /* XXX */ 1171 } 1172 datalen = port->tsolen - hlen; 1173 1174 gso_size = skb_shinfo(skb)->gso_size; 1175 gso_type = skb_shinfo(skb)->gso_type; 1176 gso_segs = skb_shinfo(skb)->gso_segs; 1177 1178 if (port->tso && gso_size < datalen) 1179 gso_segs = DIV_ROUND_UP(skb->len - hlen, datalen); 1180 1181 if (unlikely(vnet_tx_dring_avail(dr) < gso_segs)) { 1182 struct netdev_queue *txq; 1183 1184 txq = netdev_get_tx_queue(dev, port->q_index); 1185 netif_tx_stop_queue(txq); 1186 if (vnet_tx_dring_avail(dr) < skb_shinfo(skb)->gso_segs) 1187 return NETDEV_TX_BUSY; 1188 netif_tx_wake_queue(txq); 1189 } 1190 1191 maclen = skb_network_header(skb) - skb_mac_header(skb); 1192 skb_pull(skb, maclen); 1193 1194 if (port->tso && gso_size < datalen) { 1195 if (skb_unclone(skb, GFP_ATOMIC)) 1196 goto out_dropped; 1197 1198 /* segment to TSO size */ 1199 skb_shinfo(skb)->gso_size = datalen; 1200 skb_shinfo(skb)->gso_segs = gso_segs; 1201 } 1202 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO); 1203 if (IS_ERR(segs)) 1204 goto out_dropped; 1205 1206 skb_push(skb, maclen); 1207 skb_reset_mac_header(skb); 1208 1209 status = 0; 1210 while (segs) { 1211 struct sk_buff *curr = segs; 1212 1213 segs = segs->next; 1214 curr->next = NULL; 1215 if (port->tso && curr->len > dev->mtu) { 1216 skb_shinfo(curr)->gso_size = gso_size; 1217 skb_shinfo(curr)->gso_type = gso_type; 1218 skb_shinfo(curr)->gso_segs = 1219 DIV_ROUND_UP(curr->len - hlen, gso_size); 1220 } else 1221 skb_shinfo(curr)->gso_size = 0; 1222 1223 skb_push(curr, maclen); 1224 skb_reset_mac_header(curr); 1225 memcpy(skb_mac_header(curr), skb_mac_header(skb), 1226 maclen); 1227 curr->csum_start = skb_transport_header(curr) - curr->head; 1228 if (ip_hdr(curr)->protocol == IPPROTO_TCP) 1229 curr->csum_offset = offsetof(struct tcphdr, check); 1230 else if (ip_hdr(curr)->protocol == IPPROTO_UDP) 1231 curr->csum_offset = offsetof(struct udphdr, check); 1232 1233 if (!(status & NETDEV_TX_MASK)) 1234 status = vnet_start_xmit(curr, dev); 1235 if (status & NETDEV_TX_MASK) 1236 dev_kfree_skb_any(curr); 1237 } 1238 1239 if (!(status & NETDEV_TX_MASK)) 1240 dev_kfree_skb_any(skb); 1241 return status; 1242out_dropped: 1243 dev->stats.tx_dropped++; 1244 dev_kfree_skb_any(skb); 1245 return NETDEV_TX_OK; 1246} 1247 1248static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 1249{ 1250 struct vnet *vp = netdev_priv(dev); 1251 struct vnet_port *port = NULL; 1252 struct vio_dring_state *dr; 1253 struct vio_net_desc *d; 1254 unsigned int len; 1255 struct sk_buff *freeskbs = NULL; 1256 int i, err, txi; 1257 unsigned pending = 0; 1258 struct netdev_queue *txq; 1259 1260 rcu_read_lock(); 1261 port = __tx_port_find(vp, skb); 1262 if (unlikely(!port)) { 1263 rcu_read_unlock(); 1264 goto out_dropped; 1265 } 1266 1267 if (skb_is_gso(skb) && skb->len > port->tsolen) { 1268 err = vnet_handle_offloads(port, skb); 1269 rcu_read_unlock(); 1270 return err; 1271 } 1272 1273 if (!skb_is_gso(skb) && skb->len > port->rmtu) { 1274 unsigned long localmtu = port->rmtu - ETH_HLEN; 1275 1276 if (vio_version_after_eq(&port->vio, 1, 3)) 1277 localmtu -= VLAN_HLEN; 1278 1279 if (skb->protocol == htons(ETH_P_IP)) { 1280 struct flowi4 fl4; 1281 struct rtable *rt = NULL; 1282 1283 memset(&fl4, 0, sizeof(fl4)); 1284 fl4.flowi4_oif = dev->ifindex; 1285 fl4.flowi4_tos = RT_TOS(ip_hdr(skb)->tos); 1286 fl4.daddr = ip_hdr(skb)->daddr; 1287 fl4.saddr = ip_hdr(skb)->saddr; 1288 1289 rt = ip_route_output_key(dev_net(dev), &fl4); 1290 rcu_read_unlock(); 1291 if (!IS_ERR(rt)) { 1292 skb_dst_set(skb, &rt->dst); 1293 icmp_send(skb, ICMP_DEST_UNREACH, 1294 ICMP_FRAG_NEEDED, 1295 htonl(localmtu)); 1296 } 1297 } 1298#if IS_ENABLED(CONFIG_IPV6) 1299 else if (skb->protocol == htons(ETH_P_IPV6)) 1300 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, localmtu); 1301#endif 1302 goto out_dropped; 1303 } 1304 1305 skb = vnet_skb_shape(skb, 2); 1306 1307 if (unlikely(!skb)) 1308 goto out_dropped; 1309 1310 if (skb->ip_summed == CHECKSUM_PARTIAL) 1311 vnet_fullcsum(skb); 1312 1313 dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1314 i = skb_get_queue_mapping(skb); 1315 txq = netdev_get_tx_queue(dev, i); 1316 if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1317 if (!netif_tx_queue_stopped(txq)) { 1318 netif_tx_stop_queue(txq); 1319 1320 /* This is a hard error, log it. */ 1321 netdev_err(dev, "BUG! Tx Ring full when queue awake!\n"); 1322 dev->stats.tx_errors++; 1323 } 1324 rcu_read_unlock(); 1325 return NETDEV_TX_BUSY; 1326 } 1327 1328 d = vio_dring_cur(dr); 1329 1330 txi = dr->prod; 1331 1332 freeskbs = vnet_clean_tx_ring(port, &pending); 1333 1334 BUG_ON(port->tx_bufs[txi].skb); 1335 1336 len = skb->len; 1337 if (len < ETH_ZLEN) 1338 len = ETH_ZLEN; 1339 1340 err = vnet_skb_map(port->vio.lp, skb, port->tx_bufs[txi].cookies, 2, 1341 (LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW)); 1342 if (err < 0) { 1343 netdev_info(dev, "tx buffer map error %d\n", err); 1344 goto out_dropped; 1345 } 1346 1347 port->tx_bufs[txi].skb = skb; 1348 skb = NULL; 1349 port->tx_bufs[txi].ncookies = err; 1350 1351 /* We don't rely on the ACKs to free the skb in vnet_start_xmit(), 1352 * thus it is safe to not set VIO_ACK_ENABLE for each transmission: 1353 * the protocol itself does not require it as long as the peer 1354 * sends a VIO_SUBTYPE_ACK for VIO_DRING_STOPPED. 1355 * 1356 * An ACK for every packet in the ring is expensive as the 1357 * sending of LDC messages is slow and affects performance. 1358 */ 1359 d->hdr.ack = VIO_ACK_DISABLE; 1360 d->size = len; 1361 d->ncookies = port->tx_bufs[txi].ncookies; 1362 for (i = 0; i < d->ncookies; i++) 1363 d->cookies[i] = port->tx_bufs[txi].cookies[i]; 1364 if (vio_version_after_eq(&port->vio, 1, 7)) { 1365 struct vio_net_dext *dext = vio_net_ext(d); 1366 1367 memset(dext, 0, sizeof(*dext)); 1368 if (skb_is_gso(port->tx_bufs[txi].skb)) { 1369 dext->ipv4_lso_mss = skb_shinfo(port->tx_bufs[txi].skb) 1370 ->gso_size; 1371 dext->flags |= VNET_PKT_IPV4_LSO; 1372 } 1373 if (vio_version_after_eq(&port->vio, 1, 8) && 1374 !port->switch_port) { 1375 dext->flags |= VNET_PKT_HCK_IPV4_HDRCKSUM_OK; 1376 dext->flags |= VNET_PKT_HCK_FULLCKSUM_OK; 1377 } 1378 } 1379 1380 /* This has to be a non-SMP write barrier because we are writing 1381 * to memory which is shared with the peer LDOM. 1382 */ 1383 dma_wmb(); 1384 1385 d->hdr.state = VIO_DESC_READY; 1386 1387 /* Exactly one ldc "start" trigger (for dr->cons) needs to be sent 1388 * to notify the consumer that some descriptors are READY. 1389 * After that "start" trigger, no additional triggers are needed until 1390 * a DRING_STOPPED is received from the consumer. The dr->cons field 1391 * (set up by vnet_ack()) has the value of the next dring index 1392 * that has not yet been ack-ed. We send a "start" trigger here 1393 * if, and only if, start_cons is true (reset it afterward). Conversely, 1394 * vnet_ack() should check if the dring corresponding to cons 1395 * is marked READY, but start_cons was false. 1396 * If so, vnet_ack() should send out the missed "start" trigger. 1397 * 1398 * Note that the dma_wmb() above makes sure the cookies et al. are 1399 * not globally visible before the VIO_DESC_READY, and that the 1400 * stores are ordered correctly by the compiler. The consumer will 1401 * not proceed until the VIO_DESC_READY is visible assuring that 1402 * the consumer does not observe anything related to descriptors 1403 * out of order. The HV trap from the LDC start trigger is the 1404 * producer to consumer announcement that work is available to the 1405 * consumer 1406 */ 1407 if (!port->start_cons) 1408 goto ldc_start_done; /* previous trigger suffices */ 1409 1410 err = __vnet_tx_trigger(port, dr->cons); 1411 if (unlikely(err < 0)) { 1412 netdev_info(dev, "TX trigger error %d\n", err); 1413 d->hdr.state = VIO_DESC_FREE; 1414 skb = port->tx_bufs[txi].skb; 1415 port->tx_bufs[txi].skb = NULL; 1416 dev->stats.tx_carrier_errors++; 1417 goto out_dropped; 1418 } 1419 1420ldc_start_done: 1421 port->start_cons = false; 1422 1423 dev->stats.tx_packets++; 1424 dev->stats.tx_bytes += port->tx_bufs[txi].skb->len; 1425 1426 dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1); 1427 if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1428 netif_tx_stop_queue(txq); 1429 if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr)) 1430 netif_tx_wake_queue(txq); 1431 } 1432 1433 (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT); 1434 rcu_read_unlock(); 1435 1436 vnet_free_skbs(freeskbs); 1437 1438 return NETDEV_TX_OK; 1439 1440out_dropped: 1441 if (pending) 1442 (void)mod_timer(&port->clean_timer, 1443 jiffies + VNET_CLEAN_TIMEOUT); 1444 else if (port) 1445 del_timer(&port->clean_timer); 1446 if (port) 1447 rcu_read_unlock(); 1448 if (skb) 1449 dev_kfree_skb(skb); 1450 vnet_free_skbs(freeskbs); 1451 dev->stats.tx_dropped++; 1452 return NETDEV_TX_OK; 1453} 1454 1455static void vnet_tx_timeout(struct net_device *dev) 1456{ 1457 /* XXX Implement me XXX */ 1458} 1459 1460static int vnet_open(struct net_device *dev) 1461{ 1462 netif_carrier_on(dev); 1463 netif_tx_start_all_queues(dev); 1464 1465 return 0; 1466} 1467 1468static int vnet_close(struct net_device *dev) 1469{ 1470 netif_tx_stop_all_queues(dev); 1471 netif_carrier_off(dev); 1472 1473 return 0; 1474} 1475 1476static struct vnet_mcast_entry *__vnet_mc_find(struct vnet *vp, u8 *addr) 1477{ 1478 struct vnet_mcast_entry *m; 1479 1480 for (m = vp->mcast_list; m; m = m->next) { 1481 if (ether_addr_equal(m->addr, addr)) 1482 return m; 1483 } 1484 return NULL; 1485} 1486 1487static void __update_mc_list(struct vnet *vp, struct net_device *dev) 1488{ 1489 struct netdev_hw_addr *ha; 1490 1491 netdev_for_each_mc_addr(ha, dev) { 1492 struct vnet_mcast_entry *m; 1493 1494 m = __vnet_mc_find(vp, ha->addr); 1495 if (m) { 1496 m->hit = 1; 1497 continue; 1498 } 1499 1500 if (!m) { 1501 m = kzalloc(sizeof(*m), GFP_ATOMIC); 1502 if (!m) 1503 continue; 1504 memcpy(m->addr, ha->addr, ETH_ALEN); 1505 m->hit = 1; 1506 1507 m->next = vp->mcast_list; 1508 vp->mcast_list = m; 1509 } 1510 } 1511} 1512 1513static void __send_mc_list(struct vnet *vp, struct vnet_port *port) 1514{ 1515 struct vio_net_mcast_info info; 1516 struct vnet_mcast_entry *m, **pp; 1517 int n_addrs; 1518 1519 memset(&info, 0, sizeof(info)); 1520 1521 info.tag.type = VIO_TYPE_CTRL; 1522 info.tag.stype = VIO_SUBTYPE_INFO; 1523 info.tag.stype_env = VNET_MCAST_INFO; 1524 info.tag.sid = vio_send_sid(&port->vio); 1525 info.set = 1; 1526 1527 n_addrs = 0; 1528 for (m = vp->mcast_list; m; m = m->next) { 1529 if (m->sent) 1530 continue; 1531 m->sent = 1; 1532 memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1533 m->addr, ETH_ALEN); 1534 if (++n_addrs == VNET_NUM_MCAST) { 1535 info.count = n_addrs; 1536 1537 (void) vio_ldc_send(&port->vio, &info, 1538 sizeof(info)); 1539 n_addrs = 0; 1540 } 1541 } 1542 if (n_addrs) { 1543 info.count = n_addrs; 1544 (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1545 } 1546 1547 info.set = 0; 1548 1549 n_addrs = 0; 1550 pp = &vp->mcast_list; 1551 while ((m = *pp) != NULL) { 1552 if (m->hit) { 1553 m->hit = 0; 1554 pp = &m->next; 1555 continue; 1556 } 1557 1558 memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1559 m->addr, ETH_ALEN); 1560 if (++n_addrs == VNET_NUM_MCAST) { 1561 info.count = n_addrs; 1562 (void) vio_ldc_send(&port->vio, &info, 1563 sizeof(info)); 1564 n_addrs = 0; 1565 } 1566 1567 *pp = m->next; 1568 kfree(m); 1569 } 1570 if (n_addrs) { 1571 info.count = n_addrs; 1572 (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1573 } 1574} 1575 1576static void vnet_set_rx_mode(struct net_device *dev) 1577{ 1578 struct vnet *vp = netdev_priv(dev); 1579 struct vnet_port *port; 1580 1581 rcu_read_lock(); 1582 list_for_each_entry_rcu(port, &vp->port_list, list) { 1583 1584 if (port->switch_port) { 1585 __update_mc_list(vp, dev); 1586 __send_mc_list(vp, port); 1587 break; 1588 } 1589 } 1590 rcu_read_unlock(); 1591} 1592 1593static int vnet_change_mtu(struct net_device *dev, int new_mtu) 1594{ 1595 if (new_mtu < 68 || new_mtu > 65535) 1596 return -EINVAL; 1597 1598 dev->mtu = new_mtu; 1599 return 0; 1600} 1601 1602static int vnet_set_mac_addr(struct net_device *dev, void *p) 1603{ 1604 return -EINVAL; 1605} 1606 1607static void vnet_get_drvinfo(struct net_device *dev, 1608 struct ethtool_drvinfo *info) 1609{ 1610 strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); 1611 strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); 1612} 1613 1614static u32 vnet_get_msglevel(struct net_device *dev) 1615{ 1616 struct vnet *vp = netdev_priv(dev); 1617 return vp->msg_enable; 1618} 1619 1620static void vnet_set_msglevel(struct net_device *dev, u32 value) 1621{ 1622 struct vnet *vp = netdev_priv(dev); 1623 vp->msg_enable = value; 1624} 1625 1626static const struct ethtool_ops vnet_ethtool_ops = { 1627 .get_drvinfo = vnet_get_drvinfo, 1628 .get_msglevel = vnet_get_msglevel, 1629 .set_msglevel = vnet_set_msglevel, 1630 .get_link = ethtool_op_get_link, 1631}; 1632 1633static void vnet_port_free_tx_bufs(struct vnet_port *port) 1634{ 1635 struct vio_dring_state *dr; 1636 int i; 1637 1638 dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1639 1640 if (dr->base == NULL) 1641 return; 1642 1643 for (i = 0; i < VNET_TX_RING_SIZE; i++) { 1644 struct vio_net_desc *d; 1645 void *skb = port->tx_bufs[i].skb; 1646 1647 if (!skb) 1648 continue; 1649 1650 d = vio_dring_entry(dr, i); 1651 1652 ldc_unmap(port->vio.lp, 1653 port->tx_bufs[i].cookies, 1654 port->tx_bufs[i].ncookies); 1655 dev_kfree_skb(skb); 1656 port->tx_bufs[i].skb = NULL; 1657 d->hdr.state = VIO_DESC_FREE; 1658 } 1659 ldc_free_exp_dring(port->vio.lp, dr->base, 1660 (dr->entry_size * dr->num_entries), 1661 dr->cookies, dr->ncookies); 1662 dr->base = NULL; 1663 dr->entry_size = 0; 1664 dr->num_entries = 0; 1665 dr->pending = 0; 1666 dr->ncookies = 0; 1667} 1668 1669static void vnet_port_reset(struct vnet_port *port) 1670{ 1671 del_timer(&port->clean_timer); 1672 vnet_port_free_tx_bufs(port); 1673 port->rmtu = 0; 1674 port->tso = true; 1675 port->tsolen = 0; 1676} 1677 1678static int vnet_port_alloc_tx_ring(struct vnet_port *port) 1679{ 1680 struct vio_dring_state *dr; 1681 unsigned long len, elen; 1682 int i, err, ncookies; 1683 void *dring; 1684 1685 dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1686 1687 elen = sizeof(struct vio_net_desc) + 1688 sizeof(struct ldc_trans_cookie) * 2; 1689 if (vio_version_after_eq(&port->vio, 1, 7)) 1690 elen += sizeof(struct vio_net_dext); 1691 len = VNET_TX_RING_SIZE * elen; 1692 1693 ncookies = VIO_MAX_RING_COOKIES; 1694 dring = ldc_alloc_exp_dring(port->vio.lp, len, 1695 dr->cookies, &ncookies, 1696 (LDC_MAP_SHADOW | 1697 LDC_MAP_DIRECT | 1698 LDC_MAP_RW)); 1699 if (IS_ERR(dring)) { 1700 err = PTR_ERR(dring); 1701 goto err_out; 1702 } 1703 1704 dr->base = dring; 1705 dr->entry_size = elen; 1706 dr->num_entries = VNET_TX_RING_SIZE; 1707 dr->prod = dr->cons = 0; 1708 port->start_cons = true; /* need an initial trigger */ 1709 dr->pending = VNET_TX_RING_SIZE; 1710 dr->ncookies = ncookies; 1711 1712 for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 1713 struct vio_net_desc *d; 1714 1715 d = vio_dring_entry(dr, i); 1716 d->hdr.state = VIO_DESC_FREE; 1717 } 1718 return 0; 1719 1720err_out: 1721 vnet_port_free_tx_bufs(port); 1722 1723 return err; 1724} 1725 1726#ifdef CONFIG_NET_POLL_CONTROLLER 1727static void vnet_poll_controller(struct net_device *dev) 1728{ 1729 struct vnet *vp = netdev_priv(dev); 1730 struct vnet_port *port; 1731 unsigned long flags; 1732 1733 spin_lock_irqsave(&vp->lock, flags); 1734 if (!list_empty(&vp->port_list)) { 1735 port = list_entry(vp->port_list.next, struct vnet_port, list); 1736 napi_schedule(&port->napi); 1737 } 1738 spin_unlock_irqrestore(&vp->lock, flags); 1739} 1740#endif 1741static LIST_HEAD(vnet_list); 1742static DEFINE_MUTEX(vnet_list_mutex); 1743 1744static const struct net_device_ops vnet_ops = { 1745 .ndo_open = vnet_open, 1746 .ndo_stop = vnet_close, 1747 .ndo_set_rx_mode = vnet_set_rx_mode, 1748 .ndo_set_mac_address = vnet_set_mac_addr, 1749 .ndo_validate_addr = eth_validate_addr, 1750 .ndo_tx_timeout = vnet_tx_timeout, 1751 .ndo_change_mtu = vnet_change_mtu, 1752 .ndo_start_xmit = vnet_start_xmit, 1753 .ndo_select_queue = vnet_select_queue, 1754#ifdef CONFIG_NET_POLL_CONTROLLER 1755 .ndo_poll_controller = vnet_poll_controller, 1756#endif 1757}; 1758 1759static struct vnet *vnet_new(const u64 *local_mac) 1760{ 1761 struct net_device *dev; 1762 struct vnet *vp; 1763 int err, i; 1764 1765 dev = alloc_etherdev_mqs(sizeof(*vp), VNET_MAX_TXQS, 1); 1766 if (!dev) 1767 return ERR_PTR(-ENOMEM); 1768 dev->needed_headroom = VNET_PACKET_SKIP + 8; 1769 dev->needed_tailroom = 8; 1770 1771 for (i = 0; i < ETH_ALEN; i++) 1772 dev->dev_addr[i] = (*local_mac >> (5 - i) * 8) & 0xff; 1773 1774 vp = netdev_priv(dev); 1775 1776 spin_lock_init(&vp->lock); 1777 vp->dev = dev; 1778 1779 INIT_LIST_HEAD(&vp->port_list); 1780 for (i = 0; i < VNET_PORT_HASH_SIZE; i++) 1781 INIT_HLIST_HEAD(&vp->port_hash[i]); 1782 INIT_LIST_HEAD(&vp->list); 1783 vp->local_mac = *local_mac; 1784 1785 dev->netdev_ops = &vnet_ops; 1786 dev->ethtool_ops = &vnet_ethtool_ops; 1787 dev->watchdog_timeo = VNET_TX_TIMEOUT; 1788 1789 dev->hw_features = NETIF_F_TSO | NETIF_F_GSO | NETIF_F_GSO_SOFTWARE | 1790 NETIF_F_HW_CSUM | NETIF_F_SG; 1791 dev->features = dev->hw_features; 1792 1793 err = register_netdev(dev); 1794 if (err) { 1795 pr_err("Cannot register net device, aborting\n"); 1796 goto err_out_free_dev; 1797 } 1798 1799 netdev_info(dev, "Sun LDOM vnet %pM\n", dev->dev_addr); 1800 1801 list_add(&vp->list, &vnet_list); 1802 1803 return vp; 1804 1805err_out_free_dev: 1806 free_netdev(dev); 1807 1808 return ERR_PTR(err); 1809} 1810 1811static struct vnet *vnet_find_or_create(const u64 *local_mac) 1812{ 1813 struct vnet *iter, *vp; 1814 1815 mutex_lock(&vnet_list_mutex); 1816 vp = NULL; 1817 list_for_each_entry(iter, &vnet_list, list) { 1818 if (iter->local_mac == *local_mac) { 1819 vp = iter; 1820 break; 1821 } 1822 } 1823 if (!vp) 1824 vp = vnet_new(local_mac); 1825 mutex_unlock(&vnet_list_mutex); 1826 1827 return vp; 1828} 1829 1830static void vnet_cleanup(void) 1831{ 1832 struct vnet *vp; 1833 struct net_device *dev; 1834 1835 mutex_lock(&vnet_list_mutex); 1836 while (!list_empty(&vnet_list)) { 1837 vp = list_first_entry(&vnet_list, struct vnet, list); 1838 list_del(&vp->list); 1839 dev = vp->dev; 1840 /* vio_unregister_driver() should have cleaned up port_list */ 1841 BUG_ON(!list_empty(&vp->port_list)); 1842 unregister_netdev(dev); 1843 free_netdev(dev); 1844 } 1845 mutex_unlock(&vnet_list_mutex); 1846} 1847 1848static const char *local_mac_prop = "local-mac-address"; 1849 1850static struct vnet *vnet_find_parent(struct mdesc_handle *hp, 1851 u64 port_node) 1852{ 1853 const u64 *local_mac = NULL; 1854 u64 a; 1855 1856 mdesc_for_each_arc(a, hp, port_node, MDESC_ARC_TYPE_BACK) { 1857 u64 target = mdesc_arc_target(hp, a); 1858 const char *name; 1859 1860 name = mdesc_get_property(hp, target, "name", NULL); 1861 if (!name || strcmp(name, "network")) 1862 continue; 1863 1864 local_mac = mdesc_get_property(hp, target, 1865 local_mac_prop, NULL); 1866 if (local_mac) 1867 break; 1868 } 1869 if (!local_mac) 1870 return ERR_PTR(-ENODEV); 1871 1872 return vnet_find_or_create(local_mac); 1873} 1874 1875static struct ldc_channel_config vnet_ldc_cfg = { 1876 .event = vnet_event, 1877 .mtu = 64, 1878 .mode = LDC_MODE_UNRELIABLE, 1879}; 1880 1881static struct vio_driver_ops vnet_vio_ops = { 1882 .send_attr = vnet_send_attr, 1883 .handle_attr = vnet_handle_attr, 1884 .handshake_complete = vnet_handshake_complete, 1885}; 1886 1887static void print_version(void) 1888{ 1889 printk_once(KERN_INFO "%s", version); 1890} 1891 1892const char *remote_macaddr_prop = "remote-mac-address"; 1893 1894static void 1895vnet_port_add_txq(struct vnet_port *port) 1896{ 1897 struct vnet *vp = port->vp; 1898 int n; 1899 1900 n = vp->nports++; 1901 n = n & (VNET_MAX_TXQS - 1); 1902 port->q_index = n; 1903 netif_tx_wake_queue(netdev_get_tx_queue(vp->dev, port->q_index)); 1904} 1905 1906static void 1907vnet_port_rm_txq(struct vnet_port *port) 1908{ 1909 port->vp->nports--; 1910 netif_tx_stop_queue(netdev_get_tx_queue(port->vp->dev, port->q_index)); 1911} 1912 1913static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) 1914{ 1915 struct mdesc_handle *hp; 1916 struct vnet_port *port; 1917 unsigned long flags; 1918 struct vnet *vp; 1919 const u64 *rmac; 1920 int len, i, err, switch_port; 1921 1922 print_version(); 1923 1924 hp = mdesc_grab(); 1925 1926 vp = vnet_find_parent(hp, vdev->mp); 1927 if (IS_ERR(vp)) { 1928 pr_err("Cannot find port parent vnet\n"); 1929 err = PTR_ERR(vp); 1930 goto err_out_put_mdesc; 1931 } 1932 1933 rmac = mdesc_get_property(hp, vdev->mp, remote_macaddr_prop, &len); 1934 err = -ENODEV; 1935 if (!rmac) { 1936 pr_err("Port lacks %s property\n", remote_macaddr_prop); 1937 goto err_out_put_mdesc; 1938 } 1939 1940 port = kzalloc(sizeof(*port), GFP_KERNEL); 1941 err = -ENOMEM; 1942 if (!port) 1943 goto err_out_put_mdesc; 1944 1945 for (i = 0; i < ETH_ALEN; i++) 1946 port->raddr[i] = (*rmac >> (5 - i) * 8) & 0xff; 1947 1948 port->vp = vp; 1949 1950 err = vio_driver_init(&port->vio, vdev, VDEV_NETWORK, 1951 vnet_versions, ARRAY_SIZE(vnet_versions), 1952 &vnet_vio_ops, vp->dev->name); 1953 if (err) 1954 goto err_out_free_port; 1955 1956 err = vio_ldc_alloc(&port->vio, &vnet_ldc_cfg, port); 1957 if (err) 1958 goto err_out_free_port; 1959 1960 netif_napi_add(port->vp->dev, &port->napi, vnet_poll, NAPI_POLL_WEIGHT); 1961 1962 INIT_HLIST_NODE(&port->hash); 1963 INIT_LIST_HEAD(&port->list); 1964 1965 switch_port = 0; 1966 if (mdesc_get_property(hp, vdev->mp, "switch-port", NULL) != NULL) 1967 switch_port = 1; 1968 port->switch_port = switch_port; 1969 port->tso = true; 1970 port->tsolen = 0; 1971 1972 spin_lock_irqsave(&vp->lock, flags); 1973 if (switch_port) 1974 list_add_rcu(&port->list, &vp->port_list); 1975 else 1976 list_add_tail_rcu(&port->list, &vp->port_list); 1977 hlist_add_head_rcu(&port->hash, 1978 &vp->port_hash[vnet_hashfn(port->raddr)]); 1979 vnet_port_add_txq(port); 1980 spin_unlock_irqrestore(&vp->lock, flags); 1981 1982 dev_set_drvdata(&vdev->dev, port); 1983 1984 pr_info("%s: PORT ( remote-mac %pM%s )\n", 1985 vp->dev->name, port->raddr, switch_port ? " switch-port" : ""); 1986 1987 setup_timer(&port->clean_timer, vnet_clean_timer_expire, 1988 (unsigned long)port); 1989 1990 napi_enable(&port->napi); 1991 vio_port_up(&port->vio); 1992 1993 mdesc_release(hp); 1994 1995 return 0; 1996 1997err_out_free_port: 1998 kfree(port); 1999 2000err_out_put_mdesc: 2001 mdesc_release(hp); 2002 return err; 2003} 2004 2005static int vnet_port_remove(struct vio_dev *vdev) 2006{ 2007 struct vnet_port *port = dev_get_drvdata(&vdev->dev); 2008 2009 if (port) { 2010 2011 del_timer_sync(&port->vio.timer); 2012 2013 napi_disable(&port->napi); 2014 2015 list_del_rcu(&port->list); 2016 hlist_del_rcu(&port->hash); 2017 2018 synchronize_rcu(); 2019 del_timer_sync(&port->clean_timer); 2020 vnet_port_rm_txq(port); 2021 netif_napi_del(&port->napi); 2022 vnet_port_free_tx_bufs(port); 2023 vio_ldc_free(&port->vio); 2024 2025 dev_set_drvdata(&vdev->dev, NULL); 2026 2027 kfree(port); 2028 2029 } 2030 return 0; 2031} 2032 2033static const struct vio_device_id vnet_port_match[] = { 2034 { 2035 .type = "vnet-port", 2036 }, 2037 {}, 2038}; 2039MODULE_DEVICE_TABLE(vio, vnet_port_match); 2040 2041static struct vio_driver vnet_port_driver = { 2042 .id_table = vnet_port_match, 2043 .probe = vnet_port_probe, 2044 .remove = vnet_port_remove, 2045 .name = "vnet_port", 2046}; 2047 2048static int __init vnet_init(void) 2049{ 2050 return vio_register_driver(&vnet_port_driver); 2051} 2052 2053static void __exit vnet_exit(void) 2054{ 2055 vio_unregister_driver(&vnet_port_driver); 2056 vnet_cleanup(); 2057} 2058 2059module_init(vnet_init); 2060module_exit(vnet_exit); 2061