1024 	nv_wr32(info->priv, addr, data);  in gf100_grctx_mmio_item()
1030 	const struct gf100_grctx_oclass *impl = gf100_grctx_impl(info->priv);  in gf100_grctx_generate_bundle()
1043 	const struct gf100_grctx_oclass *impl = gf100_grctx_impl(info->priv);  in gf100_grctx_generate_pagepool()
1056 	struct gf100_gr_priv *priv = info->priv;  in gf100_grctx_generate_attrib()  local
1057 	const struct gf100_grctx_oclass *impl = gf100_grctx_impl(priv);  in gf100_grctx_generate_attrib()
1062 	const int b = mmio_vram(info, size * priv->tpc_total, (1 << s), access);  in gf100_grctx_generate_attrib()
1070 	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {  in gf100_grctx_generate_attrib()
1071 		for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {  in gf100_grctx_generate_attrib()
1081 gf100_grctx_generate_unkn(struct gf100_gr_priv *priv)  in gf100_grctx_generate_unkn()  argument
1086 gf100_grctx_generate_tpcid(struct gf100_gr_priv *priv)  in gf100_grctx_generate_tpcid()  argument
1091 		for (gpc = 0; gpc < priv->gpc_nr; gpc++) {  in gf100_grctx_generate_tpcid()
1092 			if (tpc < priv->tpc_nr[gpc]) {  in gf100_grctx_generate_tpcid()
1093 				nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x698), id);  in gf100_grctx_generate_tpcid()
1094 				nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x4e8), id);  in gf100_grctx_generate_tpcid()
1095 				nv_wr32(priv, GPC_UNIT(gpc, 0x0c10 + tpc * 4), id);  in gf100_grctx_generate_tpcid()
1096 				nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x088), id);  in gf100_grctx_generate_tpcid()
1100 			nv_wr32(priv, GPC_UNIT(gpc, 0x0c08), priv->tpc_nr[gpc]);  in gf100_grctx_generate_tpcid()
1101 			nv_wr32(priv, GPC_UNIT(gpc, 0x0c8c), priv->tpc_nr[gpc]);  in gf100_grctx_generate_tpcid()
1107 gf100_grctx_generate_r406028(struct gf100_gr_priv *priv)  in gf100_grctx_generate_r406028()  argument
1110 	for (i = 0; i < priv->gpc_nr; i++)  in gf100_grctx_generate_r406028()
1111 		tmp[i / 8] |= priv->tpc_nr[i] << ((i % 8) * 4);  in gf100_grctx_generate_r406028()
1113 		nv_wr32(priv, 0x406028 + (i * 4), tmp[i]);  in gf100_grctx_generate_r406028()
1114 		nv_wr32(priv, 0x405870 + (i * 4), tmp[i]);  in gf100_grctx_generate_r406028()
1119 gf100_grctx_generate_r4060a8(struct gf100_gr_priv *priv)  in gf100_grctx_generate_r4060a8()  argument
1124 	memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));  in gf100_grctx_generate_r4060a8()
1128 	for (tpc = 0; tpc < priv->tpc_total; tpc++) {  in gf100_grctx_generate_r4060a8()
1130 			gpc = (gpc + 1) % priv->gpc_nr;  in gf100_grctx_generate_r4060a8()
1137 		nv_wr32(priv, 0x4060a8 + (i * 4), ((u32 *)data)[i]);  in gf100_grctx_generate_r4060a8()
1141 gf100_grctx_generate_r418bb8(struct gf100_gr_priv *priv)  in gf100_grctx_generate_r418bb8()  argument
1149 	memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));  in gf100_grctx_generate_r418bb8()
1152 	for (tpc = 0; tpc < priv->tpc_total; tpc++) {  in gf100_grctx_generate_r418bb8()
1154 			gpc = (gpc + 1) % priv->gpc_nr;  in gf100_grctx_generate_r418bb8()
1166 	ntpcv = priv->tpc_total;  in gf100_grctx_generate_r418bb8()
1179 	nv_wr32(priv, 0x418bb8, (priv->tpc_total << 8) |  in gf100_grctx_generate_r418bb8()
1180 				 priv->magic_not_rop_nr);  in gf100_grctx_generate_r418bb8()
1182 		nv_wr32(priv, 0x418b08 + (i * 4), data[i]);  in gf100_grctx_generate_r418bb8()
1185 	nv_wr32(priv, 0x419bd0, (priv->tpc_total << 8) |  in gf100_grctx_generate_r418bb8()
1186 				 priv->magic_not_rop_nr | data2[0]);  in gf100_grctx_generate_r418bb8()
1187 	nv_wr32(priv, 0x419be4, data2[1]);  in gf100_grctx_generate_r418bb8()
1189 		nv_wr32(priv, 0x419b00 + (i * 4), data[i]);  in gf100_grctx_generate_r418bb8()
1192 	nv_wr32(priv, 0x4078bc, (priv->tpc_total << 8) |  in gf100_grctx_generate_r418bb8()
1193 				 priv->magic_not_rop_nr);  in gf100_grctx_generate_r418bb8()
1195 		nv_wr32(priv, 0x40780c + (i * 4), data[i]);  in gf100_grctx_generate_r418bb8()
1199 gf100_grctx_generate_r406800(struct gf100_gr_priv *priv)  in gf100_grctx_generate_r406800()  argument
1206 	memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));  in gf100_grctx_generate_r406800()
1207 	for (gpc = 0; gpc < priv->gpc_nr; gpc++)  in gf100_grctx_generate_r406800()
1208 		tpc_mask |= ((1ULL << priv->tpc_nr[gpc]) - 1) << (gpc * 8);  in gf100_grctx_generate_r406800()
1211 		a = (i * (priv->tpc_total - 1)) / 32;  in gf100_grctx_generate_r406800()
1215 				gpc = (gpc + 1) % priv->gpc_nr;  in gf100_grctx_generate_r406800()
1217 			tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;  in gf100_grctx_generate_r406800()
1222 		nv_wr32(priv, 0x406800 + (i * 0x20), lower_32_bits(tpc_set));  in gf100_grctx_generate_r406800()
1223 		nv_wr32(priv, 0x406c00 + (i * 0x20), lower_32_bits(tpc_set ^ tpc_mask));  in gf100_grctx_generate_r406800()
1224 		if (priv->gpc_nr > 4) {  in gf100_grctx_generate_r406800()
1225 			nv_wr32(priv, 0x406804 + (i * 0x20), upper_32_bits(tpc_set));  in gf100_grctx_generate_r406800()
1226 			nv_wr32(priv, 0x406c04 + (i * 0x20), upper_32_bits(tpc_set ^ tpc_mask));  in gf100_grctx_generate_r406800()
1232 gf100_grctx_generate_main(struct gf100_gr_priv *priv, struct gf100_grctx *info)  in gf100_grctx_generate_main()  argument
1234 	struct gf100_grctx_oclass *oclass = (void *)nv_engine(priv)->cclass;  in gf100_grctx_generate_main()
1236 	nvkm_mc(priv)->unk260(nvkm_mc(priv), 0);  in gf100_grctx_generate_main()
1238 	gf100_gr_mmio(priv, oclass->hub);  in gf100_grctx_generate_main()
1239 	gf100_gr_mmio(priv, oclass->gpc);  in gf100_grctx_generate_main()
1240 	gf100_gr_mmio(priv, oclass->zcull);  in gf100_grctx_generate_main()
1241 	gf100_gr_mmio(priv, oclass->tpc);  in gf100_grctx_generate_main()
1242 	gf100_gr_mmio(priv, oclass->ppc);  in gf100_grctx_generate_main()
1244 	nv_wr32(priv, 0x404154, 0x00000000);  in gf100_grctx_generate_main()
1249 	oclass->unkn(priv);  in gf100_grctx_generate_main()
1251 	gf100_grctx_generate_tpcid(priv);  in gf100_grctx_generate_main()
1252 	gf100_grctx_generate_r406028(priv);  in gf100_grctx_generate_main()
1253 	gf100_grctx_generate_r4060a8(priv);  in gf100_grctx_generate_main()
1254 	gf100_grctx_generate_r418bb8(priv);  in gf100_grctx_generate_main()
1255 	gf100_grctx_generate_r406800(priv);  in gf100_grctx_generate_main()
1257 	gf100_gr_icmd(priv, oclass->icmd);  in gf100_grctx_generate_main()
1258 	nv_wr32(priv, 0x404154, 0x00000400);  in gf100_grctx_generate_main()
1259 	gf100_gr_mthd(priv, oclass->mthd);  in gf100_grctx_generate_main()
1260 	nvkm_mc(priv)->unk260(nvkm_mc(priv), 1);  in gf100_grctx_generate_main()
1264 gf100_grctx_generate(struct gf100_gr_priv *priv)  in gf100_grctx_generate()  argument
1266 	struct gf100_grctx_oclass *oclass = (void *)nv_engine(priv)->cclass;  in gf100_grctx_generate()
1267 	struct nvkm_bar *bar = nvkm_bar(priv);  in gf100_grctx_generate()
1275 	ret = nvkm_gpuobj_new(nv_object(priv), NULL, 0x80000 + priv->size,  in gf100_grctx_generate()
1278 		nv_error(priv, "failed to allocate channel memory, %d\n", ret);  in gf100_grctx_generate()
1305 	nv_wr32(priv, 0x100cb8, (chan->addr + 0x1000) >> 8);  in gf100_grctx_generate()
1306 	nv_wr32(priv, 0x100cbc, 0x80000001);  in gf100_grctx_generate()
1307 	nv_wait(priv, 0x100c80, 0x00008000, 0x00008000);  in gf100_grctx_generate()
1310 	info.priv = priv;  in gf100_grctx_generate()
1311 	info.data = priv->mmio_data;  in gf100_grctx_generate()
1312 	info.mmio = priv->mmio_list;  in gf100_grctx_generate()
1317 	if (priv->firmware) {  in gf100_grctx_generate()
1318 		nv_wr32(priv, 0x409840, 0x00000030);  in gf100_grctx_generate()
1319 		nv_wr32(priv, 0x409500, 0x80000000 | chan->addr >> 12);  in gf100_grctx_generate()
1320 		nv_wr32(priv, 0x409504, 0x00000003);  in gf100_grctx_generate()
1321 		if (!nv_wait(priv, 0x409800, 0x00000010, 0x00000010))  in gf100_grctx_generate()
1322 			nv_error(priv, "load_ctx timeout\n");  in gf100_grctx_generate()
1330 		nv_wr32(priv, 0x409840, 0x80000000);  in gf100_grctx_generate()
1331 		nv_wr32(priv, 0x409500, 0x80000000 | chan->addr >> 12);  in gf100_grctx_generate()
1332 		nv_wr32(priv, 0x409504, 0x00000001);  in gf100_grctx_generate()
1333 		if (!nv_wait(priv, 0x409800, 0x80000000, 0x80000000))  in gf100_grctx_generate()
1334 			nv_error(priv, "HUB_SET_CHAN timeout\n");  in gf100_grctx_generate()
1337 	oclass->main(priv, &info);  in gf100_grctx_generate()
1342 	nv_mask(priv, 0x409b04, 0x80000000, 0x00000000);  in gf100_grctx_generate()
1343 	nv_wr32(priv, 0x409000, 0x00000100);  in gf100_grctx_generate()
1344 	if (!nv_wait(priv, 0x409b00, 0x80000000, 0x00000000)) {  in gf100_grctx_generate()
1345 		nv_error(priv, "grctx template channel unload timeout\n");  in gf100_grctx_generate()
1350 	priv->data = kmalloc(priv->size, GFP_KERNEL);  in gf100_grctx_generate()
1351 	if (priv->data) {  in gf100_grctx_generate()
1352 		for (i = 0; i < priv->size; i += 4)  in gf100_grctx_generate()
1353 			priv->data[i / 4] = nv_ro32(chan, 0x80000 + i);  in gf100_grctx_generate()