1053 static void wmb_blk(struct nfit_blk *nfit_blk)  in wmb_blk()  argument
1056 	if (nfit_blk->nvdimm_flush) {  in wmb_blk()
1065 		writeq(1, nfit_blk->nvdimm_flush);  in wmb_blk()
1071 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)  in read_blk_stat()  argument
1073 	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];  in read_blk_stat()
1074 	u64 offset = nfit_blk->stat_offset + mmio->size * bw;  in read_blk_stat()
1082 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,  in write_blk_ctl()  argument
1086 	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];  in write_blk_ctl()
1100 	offset = nfit_blk->cmd_offset + mmio->size * bw;  in write_blk_ctl()
1105 	wmb_blk(nfit_blk);  in write_blk_ctl()
1107 	if (nfit_blk->dimm_flags & ND_BLK_DCR_LATCH)  in write_blk_ctl()
1111 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,  in acpi_nfit_blk_single_io()  argument
1115 	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];  in acpi_nfit_blk_single_io()
1120 	base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES  in acpi_nfit_blk_single_io()
1122 	write_blk_ctl(nfit_blk, lane, dpa, len, rw);  in acpi_nfit_blk_single_io()
1135 			offset = base_offset + nfit_blk->bdw_offset;  in acpi_nfit_blk_single_io()
1143 			if (nfit_blk->dimm_flags & ND_BLK_READ_FLUSH)  in acpi_nfit_blk_single_io()
1156 		wmb_blk(nfit_blk);  in acpi_nfit_blk_single_io()
1158 	rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;  in acpi_nfit_blk_single_io()
1165 	struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);  in acpi_nfit_blk_region_do_io()  local
1166 	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];  in acpi_nfit_blk_region_do_io()
1167 	struct nd_region *nd_region = nfit_blk->nd_region;  in acpi_nfit_blk_region_do_io()
1175 		rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,  in acpi_nfit_blk_region_do_io()
1323 		struct nvdimm *nvdimm, struct nfit_blk *nfit_blk)  in acpi_nfit_blk_get_flags()  argument
1333 		nfit_blk->dimm_flags = flags.flags;  in acpi_nfit_blk_get_flags()
1336 		nfit_blk->dimm_flags = ND_BLK_DCR_LATCH | ND_BLK_READ_FLUSH;  in acpi_nfit_blk_get_flags()
1352 	struct nfit_blk *nfit_blk;  in acpi_nfit_blk_region_enable()  local
1367 	nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);  in acpi_nfit_blk_region_enable()
1368 	if (!nfit_blk)  in acpi_nfit_blk_region_enable()
1370 	nd_blk_region_set_provider_data(ndbr, nfit_blk);  in acpi_nfit_blk_region_enable()
1371 	nfit_blk->nd_region = to_nd_region(dev);  in acpi_nfit_blk_region_enable()
1374 	nfit_blk->bdw_offset = nfit_mem->bdw->offset;  in acpi_nfit_blk_region_enable()
1375 	mmio = &nfit_blk->mmio[BDW];  in acpi_nfit_blk_region_enable()
1396 	nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;  in acpi_nfit_blk_region_enable()
1397 	nfit_blk->stat_offset = nfit_mem->dcr->status_offset;  in acpi_nfit_blk_region_enable()
1398 	mmio = &nfit_blk->mmio[DCR];  in acpi_nfit_blk_region_enable()
1418 	rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk);  in acpi_nfit_blk_region_enable()
1427 		nfit_blk->nvdimm_flush = devm_ioremap_nocache(dev,  in acpi_nfit_blk_region_enable()
1429 		if (!nfit_blk->nvdimm_flush)  in acpi_nfit_blk_region_enable()
1433 	if (!arch_has_wmb_pmem() && !nfit_blk->nvdimm_flush)  in acpi_nfit_blk_region_enable()
1439 	if ((u32) nfit_blk->cmd_offset % mmio->line_size  in acpi_nfit_blk_region_enable()
1443 	} else if ((u32) nfit_blk->stat_offset % mmio->line_size  in acpi_nfit_blk_region_enable()
1458 	struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);  in acpi_nfit_blk_region_disable()  local
1461 	if (!nfit_blk)  in acpi_nfit_blk_region_disable()
1466 		struct nfit_blk_mmio *mmio = &nfit_blk->mmio[i];  in acpi_nfit_blk_region_disable()