269 	struct efx_nic *efx;  member
357 size_t efx_ptp_describe_stats(struct efx_nic *efx, u8 *strings)  in efx_ptp_describe_stats()  argument
359 	if (!efx->ptp_data)  in efx_ptp_describe_stats()
366 size_t efx_ptp_update_stats(struct efx_nic *efx, u64 *stats)  in efx_ptp_update_stats()  argument
373 	if (!efx->ptp_data)  in efx_ptp_update_stats()
380 		stats[i] = *(unsigned int *)((char *)efx->ptp_data +  in efx_ptp_update_stats()
390 	rc = efx_mcdi_rpc(efx, MC_CMD_PTP, inbuf, sizeof(inbuf),  in efx_ptp_update_stats()
475 static int efx_ptp_get_attributes(struct efx_nic *efx)  in efx_ptp_get_attributes()  argument
479 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_get_attributes()
490 	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_PTP, inbuf, sizeof(inbuf),  in efx_ptp_get_attributes()
497 		netif_info(efx, probe, efx->net_dev, "no PTP support\n");  in efx_ptp_get_attributes()
500 		efx_mcdi_display_error(efx, MC_CMD_PTP, sizeof(inbuf),  in efx_ptp_get_attributes()
534 static int efx_ptp_get_timestamp_corrections(struct efx_nic *efx)  in efx_ptp_get_timestamp_corrections()  argument
547 	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_PTP, inbuf, sizeof(inbuf),  in efx_ptp_get_timestamp_corrections()
550 		efx->ptp_data->ts_corrections.tx = MCDI_DWORD(outbuf,  in efx_ptp_get_timestamp_corrections()
552 		efx->ptp_data->ts_corrections.rx = MCDI_DWORD(outbuf,  in efx_ptp_get_timestamp_corrections()
554 		efx->ptp_data->ts_corrections.pps_out = MCDI_DWORD(outbuf,  in efx_ptp_get_timestamp_corrections()
556 		efx->ptp_data->ts_corrections.pps_in = MCDI_DWORD(outbuf,  in efx_ptp_get_timestamp_corrections()
559 		efx->ptp_data->ts_corrections.tx = 0;  in efx_ptp_get_timestamp_corrections()
560 		efx->ptp_data->ts_corrections.rx = 0;  in efx_ptp_get_timestamp_corrections()
561 		efx->ptp_data->ts_corrections.pps_out = 0;  in efx_ptp_get_timestamp_corrections()
562 		efx->ptp_data->ts_corrections.pps_in = 0;  in efx_ptp_get_timestamp_corrections()
564 		efx_mcdi_display_error(efx, MC_CMD_PTP, sizeof(inbuf), outbuf,  in efx_ptp_get_timestamp_corrections()
573 static int efx_ptp_enable(struct efx_nic *efx)  in efx_ptp_enable()  argument
582 		       efx->ptp_data->channel ?  in efx_ptp_enable()
583 		       efx->ptp_data->channel->channel : 0);  in efx_ptp_enable()
584 	MCDI_SET_DWORD(inbuf, PTP_IN_ENABLE_MODE, efx->ptp_data->mode);  in efx_ptp_enable()
586 	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_PTP, inbuf, sizeof(inbuf),  in efx_ptp_enable()
590 		efx_mcdi_display_error(efx, MC_CMD_PTP,  in efx_ptp_enable()
601 static int efx_ptp_disable(struct efx_nic *efx)  in efx_ptp_disable()  argument
609 	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_PTP, inbuf, sizeof(inbuf),  in efx_ptp_disable()
616 		netif_info(efx, probe, efx->net_dev, "no PTP support\n");  in efx_ptp_disable()
618 		efx_mcdi_display_error(efx, MC_CMD_PTP,  in efx_ptp_disable()
635 static void efx_ptp_handle_no_channel(struct efx_nic *efx)  in efx_ptp_handle_no_channel()  argument
637 	netif_err(efx, drv, efx->net_dev,  in efx_ptp_handle_no_channel()
645 static void efx_ptp_send_times(struct efx_nic *efx,  in efx_ptp_send_times()  argument
650 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_send_times()
677 		efx->type->ptp_write_host_time(efx, host_time);  in efx_ptp_send_times()
713 efx_ptp_process_times(struct efx_nic *efx, MCDI_DECLARE_STRUCT_PTR(synch_buf),  in efx_ptp_process_times()  argument
723 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_process_times()
774 		netif_warn(efx, drv, efx->net_dev,  in efx_ptp_process_times()
789 		netif_warn(efx, hw, efx->net_dev,  in efx_ptp_process_times()
816 static int efx_ptp_synchronize(struct efx_nic *efx, unsigned int num_readings)  in efx_ptp_synchronize()  argument
818 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_synchronize()
836 	rc = efx_mcdi_rpc_start(efx, MC_CMD_PTP, synch_buf,  in efx_ptp_synchronize()
853 		efx_ptp_send_times(efx, &last_time);  in efx_ptp_synchronize()
856 	rc = efx_mcdi_rpc_finish(efx, MC_CMD_PTP,  in efx_ptp_synchronize()
861 		rc = efx_ptp_process_times(efx, synch_buf, response_length,  in efx_ptp_synchronize()
879 static int efx_ptp_xmit_skb(struct efx_nic *efx, struct sk_buff *skb)  in efx_ptp_xmit_skb()  argument
881 	struct efx_ptp_data *ptp_data = efx->ptp_data;  in efx_ptp_xmit_skb()
905 	rc = efx_mcdi_rpc(efx, MC_CMD_PTP,  in efx_ptp_xmit_skb()
927 static void efx_ptp_drop_time_expired_events(struct efx_nic *efx)  in efx_ptp_drop_time_expired_events()  argument
929 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_drop_time_expired_events()
946 				netif_warn(efx, hw, efx->net_dev,  in efx_ptp_drop_time_expired_events()
954 static enum ptp_packet_state efx_ptp_match_rx(struct efx_nic *efx,  in efx_ptp_match_rx()  argument
957 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_match_rx()
1003 static void efx_ptp_process_events(struct efx_nic *efx, struct sk_buff_head *q)  in efx_ptp_process_events()  argument
1005 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_process_events()
1014 		} else if (efx_ptp_match_rx(efx, skb) ==  in efx_ptp_process_events()
1030 static inline void efx_ptp_process_rx(struct efx_nic *efx, struct sk_buff *skb)  in efx_ptp_process_rx()  argument
1037 static void efx_ptp_remove_multicast_filters(struct efx_nic *efx)  in efx_ptp_remove_multicast_filters()  argument
1039 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_remove_multicast_filters()
1042 		efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,  in efx_ptp_remove_multicast_filters()
1044 		efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,  in efx_ptp_remove_multicast_filters()
1050 static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)  in efx_ptp_insert_multicast_filters()  argument
1052 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_insert_multicast_filters()
1071 	rc = efx_filter_insert_filter(efx, &rxfilter, true);  in efx_ptp_insert_multicast_filters()
1085 	rc = efx_filter_insert_filter(efx, &rxfilter, true);  in efx_ptp_insert_multicast_filters()
1094 	efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,  in efx_ptp_insert_multicast_filters()
1099 static int efx_ptp_start(struct efx_nic *efx)  in efx_ptp_start()  argument
1101 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_start()
1106 	rc = efx_ptp_insert_multicast_filters(efx);  in efx_ptp_start()
1110 	rc = efx_ptp_enable(efx);  in efx_ptp_start()
1120 	efx_ptp_remove_multicast_filters(efx);  in efx_ptp_start()
1124 static int efx_ptp_stop(struct efx_nic *efx)  in efx_ptp_stop()  argument
1126 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_stop()
1134 	rc = efx_ptp_disable(efx);  in efx_ptp_stop()
1136 	efx_ptp_remove_multicast_filters(efx);  in efx_ptp_stop()
1139 	efx_ptp_deliver_rx_queue(&efx->ptp_data->rxq);  in efx_ptp_stop()
1140 	skb_queue_purge(&efx->ptp_data->txq);  in efx_ptp_stop()
1143 	spin_lock_bh(&efx->ptp_data->evt_lock);  in efx_ptp_stop()
1144 	list_for_each_safe(cursor, next, &efx->ptp_data->evt_list) {  in efx_ptp_stop()
1145 		list_move(cursor, &efx->ptp_data->evt_free_list);  in efx_ptp_stop()
1147 	spin_unlock_bh(&efx->ptp_data->evt_lock);  in efx_ptp_stop()
1152 static int efx_ptp_restart(struct efx_nic *efx)  in efx_ptp_restart()  argument
1154 	if (efx->ptp_data && efx->ptp_data->enabled)  in efx_ptp_restart()
1155 		return efx_ptp_start(efx);  in efx_ptp_restart()
1163 	struct efx_nic *efx = ptp->efx;  in efx_ptp_pps_worker()  local
1166 	if (efx_ptp_synchronize(efx, PTP_SYNC_ATTEMPTS))  in efx_ptp_pps_worker()
1178 	struct efx_nic *efx = ptp_data->efx;  in efx_ptp_worker()  local
1183 		efx_ptp_stop(efx);  in efx_ptp_worker()
1184 		efx_ptp_start(efx);  in efx_ptp_worker()
1188 	efx_ptp_drop_time_expired_events(efx);  in efx_ptp_worker()
1191 	efx_ptp_process_events(efx, &tempq);  in efx_ptp_worker()
1194 		efx_ptp_xmit_skb(efx, skb);  in efx_ptp_worker()
1197 		efx_ptp_process_rx(efx, skb);  in efx_ptp_worker()
1217 int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel)  in efx_ptp_probe()  argument
1224 	efx->ptp_data = ptp;  in efx_ptp_probe()
1225 	if (!efx->ptp_data)  in efx_ptp_probe()
1228 	ptp->efx = efx;  in efx_ptp_probe()
1230 	ptp->rx_ts_inline = efx_nic_rev(efx) >= EFX_REV_HUNT_A0;  in efx_ptp_probe()
1232 	rc = efx_nic_alloc_buffer(efx, &ptp->start, sizeof(int), GFP_KERNEL);  in efx_ptp_probe()
1255 	rc = efx_ptp_get_attributes(efx);  in efx_ptp_probe()
1260 	rc = efx_ptp_get_timestamp_corrections(efx);  in efx_ptp_probe()
1264 	if (efx->mcdi->fn_flags &  in efx_ptp_probe()
1268 						    &efx->pci_dev->dev);  in efx_ptp_probe()
1285 	ptp_clock_unregister(efx->ptp_data->phc_clock);  in efx_ptp_probe()
1288 	destroy_workqueue(efx->ptp_data->workwq);  in efx_ptp_probe()
1291 	efx_nic_free_buffer(efx, &ptp->start);  in efx_ptp_probe()
1294 	kfree(efx->ptp_data);  in efx_ptp_probe()
1295 	efx->ptp_data = NULL;  in efx_ptp_probe()
1307 	struct efx_nic *efx = channel->efx;  in efx_ptp_probe_channel()  local
1312 	return efx_ptp_probe(efx, channel);  in efx_ptp_probe_channel()
1315 void efx_ptp_remove(struct efx_nic *efx)  in efx_ptp_remove()  argument
1317 	if (!efx->ptp_data)  in efx_ptp_remove()
1320 	(void)efx_ptp_disable(efx);  in efx_ptp_remove()
1322 	cancel_work_sync(&efx->ptp_data->work);  in efx_ptp_remove()
1323 	cancel_work_sync(&efx->ptp_data->pps_work);  in efx_ptp_remove()
1325 	skb_queue_purge(&efx->ptp_data->rxq);  in efx_ptp_remove()
1326 	skb_queue_purge(&efx->ptp_data->txq);  in efx_ptp_remove()
1328 	if (efx->ptp_data->phc_clock) {  in efx_ptp_remove()
1329 		destroy_workqueue(efx->ptp_data->pps_workwq);  in efx_ptp_remove()
1330 		ptp_clock_unregister(efx->ptp_data->phc_clock);  in efx_ptp_remove()
1333 	destroy_workqueue(efx->ptp_data->workwq);  in efx_ptp_remove()
1335 	efx_nic_free_buffer(efx, &efx->ptp_data->start);  in efx_ptp_remove()
1336 	kfree(efx->ptp_data);  in efx_ptp_remove()
1341 	efx_ptp_remove(channel->efx);  in efx_ptp_remove_channel()
1347 	snprintf(buf, len, "%s-ptp", channel->efx->name);  in efx_ptp_get_channel_name()
1353 bool efx_ptp_is_ptp_tx(struct efx_nic *efx, struct sk_buff *skb)  in efx_ptp_is_ptp_tx()  argument
1355 	return efx->ptp_data &&  in efx_ptp_is_ptp_tx()
1356 		efx->ptp_data->enabled &&  in efx_ptp_is_ptp_tx()
1374 	struct efx_nic *efx = channel->efx;  in efx_ptp_rx()  local
1375 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_rx()
1459 int efx_ptp_tx(struct efx_nic *efx, struct sk_buff *skb)  in efx_ptp_tx()  argument
1461 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_tx()
1473 int efx_ptp_get_mode(struct efx_nic *efx)  in efx_ptp_get_mode()  argument
1475 	return efx->ptp_data->mode;  in efx_ptp_get_mode()
1478 int efx_ptp_change_mode(struct efx_nic *efx, bool enable_wanted,  in efx_ptp_change_mode()  argument
1481 	if ((enable_wanted != efx->ptp_data->enabled) ||  in efx_ptp_change_mode()
1482 	    (enable_wanted && (efx->ptp_data->mode != new_mode))) {  in efx_ptp_change_mode()
1487 			if (efx->ptp_data->enabled &&  in efx_ptp_change_mode()
1488 			    (efx->ptp_data->mode != new_mode)) {  in efx_ptp_change_mode()
1489 				efx->ptp_data->enabled = false;  in efx_ptp_change_mode()
1490 				rc = efx_ptp_stop(efx);  in efx_ptp_change_mode()
1499 			efx->ptp_data->mode = new_mode;  in efx_ptp_change_mode()
1500 			if (netif_running(efx->net_dev))  in efx_ptp_change_mode()
1501 				rc = efx_ptp_start(efx);  in efx_ptp_change_mode()
1503 				rc = efx_ptp_synchronize(efx,  in efx_ptp_change_mode()
1506 					efx_ptp_stop(efx);  in efx_ptp_change_mode()
1509 			rc = efx_ptp_stop(efx);  in efx_ptp_change_mode()
1515 		efx->ptp_data->enabled = enable_wanted;  in efx_ptp_change_mode()
1521 static int efx_ptp_ts_init(struct efx_nic *efx, struct hwtstamp_config *init)  in efx_ptp_ts_init()  argument
1532 	rc = efx->type->ptp_set_ts_config(efx, init);  in efx_ptp_ts_init()
1536 	efx->ptp_data->config = *init;  in efx_ptp_ts_init()
1540 void efx_ptp_get_ts_info(struct efx_nic *efx, struct ethtool_ts_info *ts_info)  in efx_ptp_get_ts_info()  argument
1542 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_get_ts_info()
1543 	struct efx_nic *primary = efx->primary;  in efx_ptp_get_ts_info()
1557 	ts_info->rx_filters = ptp->efx->type->hwtstamp_filters;  in efx_ptp_get_ts_info()
1560 int efx_ptp_set_ts_config(struct efx_nic *efx, struct ifreq *ifr)  in efx_ptp_set_ts_config()  argument
1566 	if (!efx->ptp_data)  in efx_ptp_set_ts_config()
1572 	rc = efx_ptp_ts_init(efx, &config);  in efx_ptp_set_ts_config()
1580 int efx_ptp_get_ts_config(struct efx_nic *efx, struct ifreq *ifr)  in efx_ptp_get_ts_config()  argument
1582 	if (!efx->ptp_data)  in efx_ptp_get_ts_config()
1585 	return copy_to_user(ifr->ifr_data, &efx->ptp_data->config,  in efx_ptp_get_ts_config()
1586 			    sizeof(efx->ptp_data->config)) ? -EFAULT : 0;  in efx_ptp_get_ts_config()
1589 static void ptp_event_failure(struct efx_nic *efx, int expected_frag_len)  in ptp_event_failure()  argument
1591 	struct efx_ptp_data *ptp = efx->ptp_data;  in ptp_event_failure()
1593 	netif_err(efx, hw, efx->net_dev,  in ptp_event_failure()
1604 static void ptp_event_rx(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_rx()  argument
1612 		ptp_event_failure(efx, 3);  in ptp_event_rx()
1629 		evt->hwtimestamp = efx->ptp_data->nic_to_kernel_time(  in ptp_event_rx()
1639 		netif_err(efx, rx_err, efx->net_dev, "PTP event queue overflow\n");  in ptp_event_rx()
1644 static void ptp_event_fault(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_fault()  argument
1648 		ptp_event_failure(efx, 1);  in ptp_event_fault()
1652 	netif_err(efx, hw, efx->net_dev, "PTP error %d\n", code);  in ptp_event_fault()
1655 static void ptp_event_pps(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_pps()  argument
1661 void efx_ptp_event(struct efx_nic *efx, efx_qword_t *ev)  in efx_ptp_event()  argument
1663 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_event()
1668 			netif_warn(efx, drv, efx->net_dev,  in efx_ptp_event()
1679 		netif_err(efx, hw, efx->net_dev,  in efx_ptp_event()
1689 			ptp_event_rx(efx, ptp);  in efx_ptp_event()
1692 			ptp_event_fault(efx, ptp);  in efx_ptp_event()
1695 			ptp_event_pps(efx, ptp);  in efx_ptp_event()
1698 			netif_err(efx, hw, efx->net_dev,  in efx_ptp_event()
1704 		netif_err(efx, hw, efx->net_dev,  in efx_ptp_event()
1731 static inline u32 efx_rx_buf_timestamp_minor(struct efx_nic *efx, const u8 *eh)  in efx_rx_buf_timestamp_minor()  argument
1734 	return __le32_to_cpup((const __le32 *)(eh + efx->rx_packet_ts_offset));  in efx_rx_buf_timestamp_minor()
1736 	const u8 *data = eh + efx->rx_packet_ts_offset;  in efx_rx_buf_timestamp_minor()
1747 	struct efx_nic *efx = channel->efx;  in __efx_rx_skb_attach_timestamp()  local
1752 	pkt_timestamp_minor = (efx_rx_buf_timestamp_minor(efx,  in __efx_rx_skb_attach_timestamp()
1754 			       (u32) efx->ptp_data->ts_corrections.rx) &  in __efx_rx_skb_attach_timestamp()
1784 		netif_vdbg(efx, drv, efx->net_dev,  in __efx_rx_skb_attach_timestamp()
1802 	struct efx_nic *efx = ptp_data->efx;  in efx_phc_adjfreq()  local
1821 	rc = efx_mcdi_rpc(efx, MC_CMD_PTP, inadj, sizeof(inadj),  in efx_phc_adjfreq()
1836 	struct efx_nic *efx = ptp_data->efx;  in efx_phc_adjtime()  local
1839 	efx->ptp_data->ns_to_nic_time(delta, &nic_major, &nic_minor);  in efx_phc_adjtime()
1846 	return efx_mcdi_rpc(efx, MC_CMD_PTP, inbuf, sizeof(inbuf),  in efx_phc_adjtime()
1855 	struct efx_nic *efx = ptp_data->efx;  in efx_phc_gettime()  local
1864 	rc = efx_mcdi_rpc(efx, MC_CMD_PTP, inbuf, sizeof(inbuf),  in efx_phc_gettime()
1924 void efx_ptp_defer_probe_with_channel(struct efx_nic *efx)  in efx_ptp_defer_probe_with_channel()  argument
1929 	if (efx_ptp_disable(efx) == 0)  in efx_ptp_defer_probe_with_channel()
1930 		efx->extra_channel_type[EFX_EXTRA_CHANNEL_PTP] =  in efx_ptp_defer_probe_with_channel()
1934 void efx_ptp_start_datapath(struct efx_nic *efx)  in efx_ptp_start_datapath()  argument
1936 	if (efx_ptp_restart(efx))  in efx_ptp_start_datapath()
1937 		netif_err(efx, drv, efx->net_dev, "Failed to restart PTP.\n");  in efx_ptp_start_datapath()
1939 	if (efx->type->ptp_set_ts_sync_events)  in efx_ptp_start_datapath()
1940 		efx->type->ptp_set_ts_sync_events(efx, true, true);  in efx_ptp_start_datapath()
1943 void efx_ptp_stop_datapath(struct efx_nic *efx)  in efx_ptp_stop_datapath()  argument
1946 	if (efx->type->ptp_set_ts_sync_events)  in efx_ptp_stop_datapath()
1947 		efx->type->ptp_set_ts_sync_events(efx, false, true);  in efx_ptp_stop_datapath()
1948 	efx_ptp_stop(efx);  in efx_ptp_stop_datapath()