324 static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta);
325 static int efx_phc_adjtime(struct ptp_clock_info *ptp, s64 delta);
326 static int efx_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts);
327 static int efx_phc_settime(struct ptp_clock_info *ptp,
329 static int efx_phc_enable(struct ptp_clock_info *ptp,
479 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_get_attributes()  local
506 		ptp->ns_to_nic_time = efx_ptp_ns_to_s27;  in efx_ptp_get_attributes()
507 		ptp->nic_to_kernel_time = efx_ptp_s27_to_ktime_correction;  in efx_ptp_get_attributes()
509 		ptp->ns_to_nic_time = efx_ptp_ns_to_s_ns;  in efx_ptp_get_attributes()
510 		ptp->nic_to_kernel_time = efx_ptp_s_ns_to_ktime_correction;  in efx_ptp_get_attributes()
515 	ptp->time_format = fmt;  in efx_ptp_get_attributes()
524 		ptp->min_synchronisation_ns =  in efx_ptp_get_attributes()
528 		ptp->min_synchronisation_ns = DEFAULT_MIN_SYNCHRONISATION_NS;  in efx_ptp_get_attributes()
650 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_send_times()  local
652 	int *mc_running = ptp->start.addr;  in efx_ptp_send_times()
723 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_process_times()  local
745 			&ptp->timeset[i]);  in efx_ptp_process_times()
748 			ptp->nic_to_kernel_time(0, ptp->timeset[i].wait, 0));  in efx_ptp_process_times()
749 		window = ptp->timeset[i].window;  in efx_ptp_process_times()
762 			++ptp->invalid_sync_windows;  in efx_ptp_process_times()
764 			++ptp->oversize_sync_windows;  in efx_ptp_process_times()
765 		} else if (corrected < ptp->min_synchronisation_ns) {  in efx_ptp_process_times()
766 			++ptp->undersize_sync_windows;  in efx_ptp_process_times()
785 	start_sec = ptp->timeset[last_good].host_start >> MC_NANOSECOND_BITS;  in efx_ptp_process_times()
796 		(ptp->timeset[last_good].host_start & MC_NANOSECOND_MASK);  in efx_ptp_process_times()
802 	mc_time = ptp->nic_to_kernel_time(ptp->timeset[last_good].major,  in efx_ptp_process_times()
803 					  ptp->timeset[last_good].minor, 0);  in efx_ptp_process_times()
809 	ptp->host_time_pps = *last_time;  in efx_ptp_process_times()
810 	pps_sub_ts(&ptp->host_time_pps, delta);  in efx_ptp_process_times()
818 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_synchronize()  local
825 	int *start = ptp->start.addr;  in efx_ptp_synchronize()
832 		       ptp->start.dma_addr);  in efx_ptp_synchronize()
848 		++ptp->fast_syncs;  in efx_ptp_synchronize()
850 		++ptp->sync_timeouts;  in efx_ptp_synchronize()
864 			++ptp->good_syncs;  in efx_ptp_synchronize()
866 			++ptp->no_time_syncs;  in efx_ptp_synchronize()
873 		++ptp->bad_syncs;  in efx_ptp_synchronize()
929 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_drop_time_expired_events()  local
933 	if (ptp->rx_ts_inline)  in efx_ptp_drop_time_expired_events()
937 	spin_lock_bh(&ptp->evt_lock);  in efx_ptp_drop_time_expired_events()
938 	if (!list_empty(&ptp->evt_list)) {  in efx_ptp_drop_time_expired_events()
939 		list_for_each_safe(cursor, next, &ptp->evt_list) {  in efx_ptp_drop_time_expired_events()
945 				list_move(&evt->link, &ptp->evt_free_list);  in efx_ptp_drop_time_expired_events()
951 	spin_unlock_bh(&ptp->evt_lock);  in efx_ptp_drop_time_expired_events()
957 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_match_rx()  local
964 	WARN_ON_ONCE(ptp->rx_ts_inline);  in efx_ptp_match_rx()
966 	spin_lock_bh(&ptp->evt_lock);  in efx_ptp_match_rx()
967 	evts_waiting = !list_empty(&ptp->evt_list);  in efx_ptp_match_rx()
968 	spin_unlock_bh(&ptp->evt_lock);  in efx_ptp_match_rx()
975 	spin_lock_bh(&ptp->evt_lock);  in efx_ptp_match_rx()
976 	list_for_each_safe(cursor, next, &ptp->evt_list) {  in efx_ptp_match_rx()
990 			list_move(&evt->link, &ptp->evt_free_list);  in efx_ptp_match_rx()
994 	spin_unlock_bh(&ptp->evt_lock);  in efx_ptp_match_rx()
1005 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_process_events()  local
1008 	while ((skb = skb_dequeue(&ptp->rxq))) {  in efx_ptp_process_events()
1019 			++ptp->rx_no_timestamp;  in efx_ptp_process_events()
1023 			skb_queue_head(&ptp->rxq, skb);  in efx_ptp_process_events()
1039 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_remove_multicast_filters()  local
1041 	if (ptp->rxfilter_installed) {  in efx_ptp_remove_multicast_filters()
1043 					  ptp->rxfilter_general);  in efx_ptp_remove_multicast_filters()
1045 					  ptp->rxfilter_event);  in efx_ptp_remove_multicast_filters()
1046 		ptp->rxfilter_installed = false;  in efx_ptp_remove_multicast_filters()
1052 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_insert_multicast_filters()  local
1056 	if (!ptp->channel || ptp->rxfilter_installed)  in efx_ptp_insert_multicast_filters()
1064 				   efx_channel_get_rx_queue(ptp->channel)));  in efx_ptp_insert_multicast_filters()
1074 	ptp->rxfilter_event = rc;  in efx_ptp_insert_multicast_filters()
1078 				   efx_channel_get_rx_queue(ptp->channel)));  in efx_ptp_insert_multicast_filters()
1088 	ptp->rxfilter_general = rc;  in efx_ptp_insert_multicast_filters()
1090 	ptp->rxfilter_installed = true;  in efx_ptp_insert_multicast_filters()
1095 				  ptp->rxfilter_event);  in efx_ptp_insert_multicast_filters()
1101 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_start()  local
1104 	ptp->reset_required = false;  in efx_ptp_start()
1114 	ptp->evt_frag_idx = 0;  in efx_ptp_start()
1115 	ptp->current_adjfreq = 0;  in efx_ptp_start()
1126 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_stop()  local
1131 	if (ptp == NULL)  in efx_ptp_stop()
1161 	struct efx_ptp_data *ptp =  in efx_ptp_pps_worker()  local
1163 	struct efx_nic *efx = ptp->efx;  in efx_ptp_pps_worker()
1170 	ptp_evt.pps_times = ptp->host_time_pps;  in efx_ptp_pps_worker()
1171 	ptp_clock_event(ptp->phc_clock, &ptp_evt);  in efx_ptp_pps_worker()
1219 	struct efx_ptp_data *ptp;  in efx_ptp_probe()  local
1223 	ptp = kzalloc(sizeof(struct efx_ptp_data), GFP_KERNEL);  in efx_ptp_probe()
1224 	efx->ptp_data = ptp;  in efx_ptp_probe()
1228 	ptp->efx = efx;  in efx_ptp_probe()
1229 	ptp->channel = channel;  in efx_ptp_probe()
1230 	ptp->rx_ts_inline = efx_nic_rev(efx) >= EFX_REV_HUNT_A0;  in efx_ptp_probe()
1232 	rc = efx_nic_alloc_buffer(efx, &ptp->start, sizeof(int), GFP_KERNEL);  in efx_ptp_probe()
1236 	skb_queue_head_init(&ptp->rxq);  in efx_ptp_probe()
1237 	skb_queue_head_init(&ptp->txq);  in efx_ptp_probe()
1238 	ptp->workwq = create_singlethread_workqueue("sfc_ptp");  in efx_ptp_probe()
1239 	if (!ptp->workwq) {  in efx_ptp_probe()
1244 	INIT_WORK(&ptp->work, efx_ptp_worker);  in efx_ptp_probe()
1245 	ptp->config.flags = 0;  in efx_ptp_probe()
1246 	ptp->config.tx_type = HWTSTAMP_TX_OFF;  in efx_ptp_probe()
1247 	ptp->config.rx_filter = HWTSTAMP_FILTER_NONE;  in efx_ptp_probe()
1248 	INIT_LIST_HEAD(&ptp->evt_list);  in efx_ptp_probe()
1249 	INIT_LIST_HEAD(&ptp->evt_free_list);  in efx_ptp_probe()
1250 	spin_lock_init(&ptp->evt_lock);  in efx_ptp_probe()
1252 		list_add(&ptp->rx_evts[pos].link, &ptp->evt_free_list);  in efx_ptp_probe()
1266 		ptp->phc_clock_info = efx_phc_clock_info;  in efx_ptp_probe()
1267 		ptp->phc_clock = ptp_clock_register(&ptp->phc_clock_info,  in efx_ptp_probe()
1269 		if (IS_ERR(ptp->phc_clock)) {  in efx_ptp_probe()
1270 			rc = PTR_ERR(ptp->phc_clock);  in efx_ptp_probe()
1274 		INIT_WORK(&ptp->pps_work, efx_ptp_pps_worker);  in efx_ptp_probe()
1275 		ptp->pps_workwq = create_singlethread_workqueue("sfc_pps");  in efx_ptp_probe()
1276 		if (!ptp->pps_workwq) {  in efx_ptp_probe()
1281 	ptp->nic_ts_enabled = false;  in efx_ptp_probe()
1291 	efx_nic_free_buffer(efx, &ptp->start);  in efx_ptp_probe()
1375 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_rx()  local
1384 	if (ptp->mode == MC_CMD_PTP_MODE_V1) {  in efx_ptp_rx()
1417 		if (ptp->mode == MC_CMD_PTP_MODE_V2) {  in efx_ptp_rx()
1421 			BUG_ON(ptp->mode != MC_CMD_PTP_MODE_V2_ENHANCED);  in efx_ptp_rx()
1449 	skb_queue_tail(&ptp->rxq, skb);  in efx_ptp_rx()
1450 	queue_work(ptp->workwq, &ptp->work);  in efx_ptp_rx()
1461 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_tx()  local
1463 	skb_queue_tail(&ptp->txq, skb);  in efx_ptp_tx()
1468 	queue_work(ptp->workwq, &ptp->work);  in efx_ptp_tx()
1542 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_get_ts_info()  local
1547 	if (!ptp)  in efx_ptp_get_ts_info()
1557 	ts_info->rx_filters = ptp->efx->type->hwtstamp_filters;  in efx_ptp_get_ts_info()
1591 	struct efx_ptp_data *ptp = efx->ptp_data;  in ptp_event_failure()  local
1595 		ptp->evt_frag_idx, expected_frag_len);  in ptp_event_failure()
1596 	ptp->reset_required = true;  in ptp_event_failure()
1597 	queue_work(ptp->workwq, &ptp->work);  in ptp_event_failure()
1604 static void ptp_event_rx(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_rx()  argument
1608 	if (WARN_ON_ONCE(ptp->rx_ts_inline))  in ptp_event_rx()
1611 	if (ptp->evt_frag_idx != 3) {  in ptp_event_rx()
1616 	spin_lock_bh(&ptp->evt_lock);  in ptp_event_rx()
1617 	if (!list_empty(&ptp->evt_free_list)) {  in ptp_event_rx()
1618 		evt = list_first_entry(&ptp->evt_free_list,  in ptp_event_rx()
1622 		evt->seq0 = EFX_QWORD_FIELD(ptp->evt_frags[2], MCDI_EVENT_DATA);  in ptp_event_rx()
1623 		evt->seq1 = (EFX_QWORD_FIELD(ptp->evt_frags[2],  in ptp_event_rx()
1625 			     (EFX_QWORD_FIELD(ptp->evt_frags[1],  in ptp_event_rx()
1627 			     (EFX_QWORD_FIELD(ptp->evt_frags[0],  in ptp_event_rx()
1630 			EFX_QWORD_FIELD(ptp->evt_frags[0], MCDI_EVENT_DATA),  in ptp_event_rx()
1631 			EFX_QWORD_FIELD(ptp->evt_frags[1], MCDI_EVENT_DATA),  in ptp_event_rx()
1632 			ptp->ts_corrections.rx);  in ptp_event_rx()
1634 		list_add_tail(&evt->link, &ptp->evt_list);  in ptp_event_rx()
1636 		queue_work(ptp->workwq, &ptp->work);  in ptp_event_rx()
1641 	spin_unlock_bh(&ptp->evt_lock);  in ptp_event_rx()
1644 static void ptp_event_fault(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_fault()  argument
1646 	int code = EFX_QWORD_FIELD(ptp->evt_frags[0], MCDI_EVENT_DATA);  in ptp_event_fault()
1647 	if (ptp->evt_frag_idx != 1) {  in ptp_event_fault()
1655 static void ptp_event_pps(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_pps()  argument
1657 	if (ptp->nic_ts_enabled)  in ptp_event_pps()
1658 		queue_work(ptp->pps_workwq, &ptp->pps_work);  in ptp_event_pps()
1663 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_event()  local
1666 	if (!ptp) {  in efx_ptp_event()
1673 	if (!ptp->enabled)  in efx_ptp_event()
1676 	if (ptp->evt_frag_idx == 0) {  in efx_ptp_event()
1677 		ptp->evt_code = code;  in efx_ptp_event()
1678 	} else if (ptp->evt_code != code) {  in efx_ptp_event()
1681 		ptp->evt_frag_idx = 0;  in efx_ptp_event()
1684 	ptp->evt_frags[ptp->evt_frag_idx++] = *ev;  in efx_ptp_event()
1689 			ptp_event_rx(efx, ptp);  in efx_ptp_event()
1692 			ptp_event_fault(efx, ptp);  in efx_ptp_event()
1695 			ptp_event_pps(efx, ptp);  in efx_ptp_event()
1702 		ptp->evt_frag_idx = 0;  in efx_ptp_event()
1703 	} else if (MAX_EVENT_FRAGS == ptp->evt_frag_idx) {  in efx_ptp_event()
1706 		ptp->evt_frag_idx = 0;  in efx_ptp_event()
1797 static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta)  in efx_phc_adjfreq()  argument
1799 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_adjfreq()
1830 static int efx_phc_adjtime(struct ptp_clock_info *ptp, s64 delta)  in efx_phc_adjtime()  argument
1833 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_adjtime()
1850 static int efx_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts)  in efx_phc_gettime()  argument
1852 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_gettime()
1876 static int efx_phc_settime(struct ptp_clock_info *ptp,  in efx_phc_settime()  argument
1887 	rc = efx_phc_gettime(ptp, &time_now);  in efx_phc_settime()
1893 	rc = efx_phc_adjtime(ptp, timespec64_to_ns(&delta));  in efx_phc_settime()
1900 static int efx_phc_enable(struct ptp_clock_info *ptp,  in efx_phc_enable()  argument
1904 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_enable()