324 static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta);
325 static int efx_phc_adjtime(struct ptp_clock_info *ptp, s64 delta);
326 static int efx_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts);
327 static int efx_phc_settime(struct ptp_clock_info *ptp,
329 static int efx_phc_enable(struct ptp_clock_info *ptp,
482 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_get_attributes()  local
503 		ptp->ns_to_nic_time = efx_ptp_ns_to_s27;  in efx_ptp_get_attributes()
504 		ptp->nic_to_kernel_time = efx_ptp_s27_to_ktime_correction;  in efx_ptp_get_attributes()
506 		ptp->ns_to_nic_time = efx_ptp_ns_to_s_ns;  in efx_ptp_get_attributes()
507 		ptp->nic_to_kernel_time = efx_ptp_s_ns_to_ktime_correction;  in efx_ptp_get_attributes()
512 	ptp->time_format = fmt;  in efx_ptp_get_attributes()
521 		ptp->min_synchronisation_ns =  in efx_ptp_get_attributes()
525 		ptp->min_synchronisation_ns = DEFAULT_MIN_SYNCHRONISATION_NS;  in efx_ptp_get_attributes()
640 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_send_times()  local
642 	int *mc_running = ptp->start.addr;  in efx_ptp_send_times()
713 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_process_times()  local
735 			&ptp->timeset[i]);  in efx_ptp_process_times()
738 			ptp->nic_to_kernel_time(0, ptp->timeset[i].wait, 0));  in efx_ptp_process_times()
739 		window = ptp->timeset[i].window;  in efx_ptp_process_times()
752 			++ptp->invalid_sync_windows;  in efx_ptp_process_times()
754 			++ptp->oversize_sync_windows;  in efx_ptp_process_times()
755 		} else if (corrected < ptp->min_synchronisation_ns) {  in efx_ptp_process_times()
756 			++ptp->undersize_sync_windows;  in efx_ptp_process_times()
775 	start_sec = ptp->timeset[last_good].host_start >> MC_NANOSECOND_BITS;  in efx_ptp_process_times()
786 		(ptp->timeset[last_good].host_start & MC_NANOSECOND_MASK);  in efx_ptp_process_times()
792 	mc_time = ptp->nic_to_kernel_time(ptp->timeset[last_good].major,  in efx_ptp_process_times()
793 					  ptp->timeset[last_good].minor, 0);  in efx_ptp_process_times()
799 	ptp->host_time_pps = *last_time;  in efx_ptp_process_times()
800 	pps_sub_ts(&ptp->host_time_pps, delta);  in efx_ptp_process_times()
808 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_synchronize()  local
815 	int *start = ptp->start.addr;  in efx_ptp_synchronize()
822 		       ptp->start.dma_addr);  in efx_ptp_synchronize()
838 		++ptp->fast_syncs;  in efx_ptp_synchronize()
840 		++ptp->sync_timeouts;  in efx_ptp_synchronize()
854 			++ptp->good_syncs;  in efx_ptp_synchronize()
856 			++ptp->no_time_syncs;  in efx_ptp_synchronize()
863 		++ptp->bad_syncs;  in efx_ptp_synchronize()
919 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_drop_time_expired_events()  local
923 	if (ptp->rx_ts_inline)  in efx_ptp_drop_time_expired_events()
927 	spin_lock_bh(&ptp->evt_lock);  in efx_ptp_drop_time_expired_events()
928 	if (!list_empty(&ptp->evt_list)) {  in efx_ptp_drop_time_expired_events()
929 		list_for_each_safe(cursor, next, &ptp->evt_list) {  in efx_ptp_drop_time_expired_events()
935 				list_move(&evt->link, &ptp->evt_free_list);  in efx_ptp_drop_time_expired_events()
941 	spin_unlock_bh(&ptp->evt_lock);  in efx_ptp_drop_time_expired_events()
947 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_match_rx()  local
954 	WARN_ON_ONCE(ptp->rx_ts_inline);  in efx_ptp_match_rx()
956 	spin_lock_bh(&ptp->evt_lock);  in efx_ptp_match_rx()
957 	evts_waiting = !list_empty(&ptp->evt_list);  in efx_ptp_match_rx()
958 	spin_unlock_bh(&ptp->evt_lock);  in efx_ptp_match_rx()
965 	spin_lock_bh(&ptp->evt_lock);  in efx_ptp_match_rx()
966 	list_for_each_safe(cursor, next, &ptp->evt_list) {  in efx_ptp_match_rx()
980 			list_move(&evt->link, &ptp->evt_free_list);  in efx_ptp_match_rx()
984 	spin_unlock_bh(&ptp->evt_lock);  in efx_ptp_match_rx()
995 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_process_events()  local
998 	while ((skb = skb_dequeue(&ptp->rxq))) {  in efx_ptp_process_events()
1009 			++ptp->rx_no_timestamp;  in efx_ptp_process_events()
1013 			skb_queue_head(&ptp->rxq, skb);  in efx_ptp_process_events()
1029 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_remove_multicast_filters()  local
1031 	if (ptp->rxfilter_installed) {  in efx_ptp_remove_multicast_filters()
1033 					  ptp->rxfilter_general);  in efx_ptp_remove_multicast_filters()
1035 					  ptp->rxfilter_event);  in efx_ptp_remove_multicast_filters()
1036 		ptp->rxfilter_installed = false;  in efx_ptp_remove_multicast_filters()
1042 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_insert_multicast_filters()  local
1046 	if (!ptp->channel || ptp->rxfilter_installed)  in efx_ptp_insert_multicast_filters()
1054 				   efx_channel_get_rx_queue(ptp->channel)));  in efx_ptp_insert_multicast_filters()
1064 	ptp->rxfilter_event = rc;  in efx_ptp_insert_multicast_filters()
1068 				   efx_channel_get_rx_queue(ptp->channel)));  in efx_ptp_insert_multicast_filters()
1078 	ptp->rxfilter_general = rc;  in efx_ptp_insert_multicast_filters()
1080 	ptp->rxfilter_installed = true;  in efx_ptp_insert_multicast_filters()
1085 				  ptp->rxfilter_event);  in efx_ptp_insert_multicast_filters()
1091 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_start()  local
1094 	ptp->reset_required = false;  in efx_ptp_start()
1104 	ptp->evt_frag_idx = 0;  in efx_ptp_start()
1105 	ptp->current_adjfreq = 0;  in efx_ptp_start()
1116 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_stop()  local
1121 	if (ptp == NULL)  in efx_ptp_stop()
1151 	struct efx_ptp_data *ptp =  in efx_ptp_pps_worker()  local
1153 	struct efx_nic *efx = ptp->efx;  in efx_ptp_pps_worker()
1160 	ptp_evt.pps_times = ptp->host_time_pps;  in efx_ptp_pps_worker()
1161 	ptp_clock_event(ptp->phc_clock, &ptp_evt);  in efx_ptp_pps_worker()
1209 	struct efx_ptp_data *ptp;  in efx_ptp_probe()  local
1213 	ptp = kzalloc(sizeof(struct efx_ptp_data), GFP_KERNEL);  in efx_ptp_probe()
1214 	efx->ptp_data = ptp;  in efx_ptp_probe()
1218 	ptp->efx = efx;  in efx_ptp_probe()
1219 	ptp->channel = channel;  in efx_ptp_probe()
1220 	ptp->rx_ts_inline = efx_nic_rev(efx) >= EFX_REV_HUNT_A0;  in efx_ptp_probe()
1222 	rc = efx_nic_alloc_buffer(efx, &ptp->start, sizeof(int), GFP_KERNEL);  in efx_ptp_probe()
1226 	skb_queue_head_init(&ptp->rxq);  in efx_ptp_probe()
1227 	skb_queue_head_init(&ptp->txq);  in efx_ptp_probe()
1228 	ptp->workwq = create_singlethread_workqueue("sfc_ptp");  in efx_ptp_probe()
1229 	if (!ptp->workwq) {  in efx_ptp_probe()
1234 	INIT_WORK(&ptp->work, efx_ptp_worker);  in efx_ptp_probe()
1235 	ptp->config.flags = 0;  in efx_ptp_probe()
1236 	ptp->config.tx_type = HWTSTAMP_TX_OFF;  in efx_ptp_probe()
1237 	ptp->config.rx_filter = HWTSTAMP_FILTER_NONE;  in efx_ptp_probe()
1238 	INIT_LIST_HEAD(&ptp->evt_list);  in efx_ptp_probe()
1239 	INIT_LIST_HEAD(&ptp->evt_free_list);  in efx_ptp_probe()
1240 	spin_lock_init(&ptp->evt_lock);  in efx_ptp_probe()
1242 		list_add(&ptp->rx_evts[pos].link, &ptp->evt_free_list);  in efx_ptp_probe()
1256 		ptp->phc_clock_info = efx_phc_clock_info;  in efx_ptp_probe()
1257 		ptp->phc_clock = ptp_clock_register(&ptp->phc_clock_info,  in efx_ptp_probe()
1259 		if (IS_ERR(ptp->phc_clock)) {  in efx_ptp_probe()
1260 			rc = PTR_ERR(ptp->phc_clock);  in efx_ptp_probe()
1264 		INIT_WORK(&ptp->pps_work, efx_ptp_pps_worker);  in efx_ptp_probe()
1265 		ptp->pps_workwq = create_singlethread_workqueue("sfc_pps");  in efx_ptp_probe()
1266 		if (!ptp->pps_workwq) {  in efx_ptp_probe()
1271 	ptp->nic_ts_enabled = false;  in efx_ptp_probe()
1281 	efx_nic_free_buffer(efx, &ptp->start);  in efx_ptp_probe()
1365 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_rx()  local
1374 	if (ptp->mode == MC_CMD_PTP_MODE_V1) {  in efx_ptp_rx()
1407 		if (ptp->mode == MC_CMD_PTP_MODE_V2) {  in efx_ptp_rx()
1411 			BUG_ON(ptp->mode != MC_CMD_PTP_MODE_V2_ENHANCED);  in efx_ptp_rx()
1439 	skb_queue_tail(&ptp->rxq, skb);  in efx_ptp_rx()
1440 	queue_work(ptp->workwq, &ptp->work);  in efx_ptp_rx()
1451 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_tx()  local
1453 	skb_queue_tail(&ptp->txq, skb);  in efx_ptp_tx()
1458 	queue_work(ptp->workwq, &ptp->work);  in efx_ptp_tx()
1532 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_get_ts_info()  local
1537 	if (!ptp)  in efx_ptp_get_ts_info()
1547 	ts_info->rx_filters = ptp->efx->type->hwtstamp_filters;  in efx_ptp_get_ts_info()
1581 	struct efx_ptp_data *ptp = efx->ptp_data;  in ptp_event_failure()  local
1585 		ptp->evt_frag_idx, expected_frag_len);  in ptp_event_failure()
1586 	ptp->reset_required = true;  in ptp_event_failure()
1587 	queue_work(ptp->workwq, &ptp->work);  in ptp_event_failure()
1594 static void ptp_event_rx(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_rx()  argument
1598 	if (WARN_ON_ONCE(ptp->rx_ts_inline))  in ptp_event_rx()
1601 	if (ptp->evt_frag_idx != 3) {  in ptp_event_rx()
1606 	spin_lock_bh(&ptp->evt_lock);  in ptp_event_rx()
1607 	if (!list_empty(&ptp->evt_free_list)) {  in ptp_event_rx()
1608 		evt = list_first_entry(&ptp->evt_free_list,  in ptp_event_rx()
1612 		evt->seq0 = EFX_QWORD_FIELD(ptp->evt_frags[2], MCDI_EVENT_DATA);  in ptp_event_rx()
1613 		evt->seq1 = (EFX_QWORD_FIELD(ptp->evt_frags[2],  in ptp_event_rx()
1615 			     (EFX_QWORD_FIELD(ptp->evt_frags[1],  in ptp_event_rx()
1617 			     (EFX_QWORD_FIELD(ptp->evt_frags[0],  in ptp_event_rx()
1620 			EFX_QWORD_FIELD(ptp->evt_frags[0], MCDI_EVENT_DATA),  in ptp_event_rx()
1621 			EFX_QWORD_FIELD(ptp->evt_frags[1], MCDI_EVENT_DATA),  in ptp_event_rx()
1622 			ptp->ts_corrections.rx);  in ptp_event_rx()
1624 		list_add_tail(&evt->link, &ptp->evt_list);  in ptp_event_rx()
1626 		queue_work(ptp->workwq, &ptp->work);  in ptp_event_rx()
1631 	spin_unlock_bh(&ptp->evt_lock);  in ptp_event_rx()
1634 static void ptp_event_fault(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_fault()  argument
1636 	int code = EFX_QWORD_FIELD(ptp->evt_frags[0], MCDI_EVENT_DATA);  in ptp_event_fault()
1637 	if (ptp->evt_frag_idx != 1) {  in ptp_event_fault()
1645 static void ptp_event_pps(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_pps()  argument
1647 	if (ptp->nic_ts_enabled)  in ptp_event_pps()
1648 		queue_work(ptp->pps_workwq, &ptp->pps_work);  in ptp_event_pps()
1653 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_event()  local
1656 	if (!ptp) {  in efx_ptp_event()
1663 	if (!ptp->enabled)  in efx_ptp_event()
1666 	if (ptp->evt_frag_idx == 0) {  in efx_ptp_event()
1667 		ptp->evt_code = code;  in efx_ptp_event()
1668 	} else if (ptp->evt_code != code) {  in efx_ptp_event()
1671 		ptp->evt_frag_idx = 0;  in efx_ptp_event()
1674 	ptp->evt_frags[ptp->evt_frag_idx++] = *ev;  in efx_ptp_event()
1679 			ptp_event_rx(efx, ptp);  in efx_ptp_event()
1682 			ptp_event_fault(efx, ptp);  in efx_ptp_event()
1685 			ptp_event_pps(efx, ptp);  in efx_ptp_event()
1692 		ptp->evt_frag_idx = 0;  in efx_ptp_event()
1693 	} else if (MAX_EVENT_FRAGS == ptp->evt_frag_idx) {  in efx_ptp_event()
1696 		ptp->evt_frag_idx = 0;  in efx_ptp_event()
1787 static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta)  in efx_phc_adjfreq()  argument
1789 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_adjfreq()
1820 static int efx_phc_adjtime(struct ptp_clock_info *ptp, s64 delta)  in efx_phc_adjtime()  argument
1823 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_adjtime()
1840 static int efx_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts)  in efx_phc_gettime()  argument
1842 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_gettime()
1866 static int efx_phc_settime(struct ptp_clock_info *ptp,  in efx_phc_settime()  argument
1877 	rc = efx_phc_gettime(ptp, &time_now);  in efx_phc_settime()
1883 	rc = efx_phc_adjtime(ptp, timespec64_to_ns(&delta));  in efx_phc_settime()
1890 static int efx_phc_enable(struct ptp_clock_info *ptp,  in efx_phc_enable()  argument
1894 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_enable()