96 gk104_fifo_runlist_update(struct gk104_fifo_priv *priv, u32 engine)  in gk104_fifo_runlist_update()  argument
98 	struct nvkm_bar *bar = nvkm_bar(priv);  in gk104_fifo_runlist_update()
99 	struct gk104_fifo_engn *engn = &priv->engine[engine];  in gk104_fifo_runlist_update()
103 	mutex_lock(&nv_subdev(priv)->mutex);  in gk104_fifo_runlist_update()
107 	for (i = 0, p = 0; i < priv->base.max; i++) {  in gk104_fifo_runlist_update()
108 		struct gk104_fifo_chan *chan = (void *)priv->base.channel[i];  in gk104_fifo_runlist_update()
117 	nv_wr32(priv, 0x002270, cur->addr >> 12);  in gk104_fifo_runlist_update()
118 	nv_wr32(priv, 0x002274, (engine << 20) | (p >> 3));  in gk104_fifo_runlist_update()
120 	if (wait_event_timeout(engn->wait, !(nv_rd32(priv, 0x002284 +  in gk104_fifo_runlist_update()
123 		nv_error(priv, "runlist %d update timeout\n", engine);  in gk104_fifo_runlist_update()
124 	mutex_unlock(&nv_subdev(priv)->mutex);  in gk104_fifo_runlist_update()
173 	struct gk104_fifo_priv *priv = (void *)parent->engine;  in gk104_fifo_context_detach()  local
191 	nv_wr32(priv, 0x002634, chan->base.chid);  in gk104_fifo_context_detach()
192 	if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) {  in gk104_fifo_context_detach()
193 		nv_error(priv, "channel %d [%s] kick timeout\n",  in gk104_fifo_context_detach()
217 	struct gk104_fifo_priv *priv = (void *)engine;  in gk104_fifo_chan_ctor()  local
242 		nv_error(priv, "unsupported engines 0x%08x\n", args->v0.engine);  in gk104_fifo_chan_ctor()
247 				       priv->user.bar.offset, 0x200,  in gk104_fifo_chan_ctor()
265 		nv_wo32(priv->user.mem, usermem + i, 0x00000000);  in gk104_fifo_chan_ctor()
267 	nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem));  in gk104_fifo_chan_ctor()
268 	nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem));  in gk104_fifo_chan_ctor()
289 	struct gk104_fifo_priv *priv = (void *)object->engine;  in gk104_fifo_chan_init()  local
298 	nv_mask(priv, 0x800004 + (chid * 8), 0x000f0000, chan->engine << 16);  in gk104_fifo_chan_init()
299 	nv_wr32(priv, 0x800000 + (chid * 8), 0x80000000 | base->addr >> 12);  in gk104_fifo_chan_init()
302 		nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400);  in gk104_fifo_chan_init()
303 		gk104_fifo_runlist_update(priv, chan->engine);  in gk104_fifo_chan_init()
304 		nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400);  in gk104_fifo_chan_init()
313 	struct gk104_fifo_priv *priv = (void *)object->engine;  in gk104_fifo_chan_fini()  local
318 		nv_mask(priv, 0x800004 + (chid * 8), 0x00000800, 0x00000800);  in gk104_fifo_chan_fini()
319 		gk104_fifo_runlist_update(priv, chan->engine);  in gk104_fifo_chan_fini()
322 	nv_wr32(priv, 0x800000 + (chid * 8), 0x00000000);  in gk104_fifo_chan_fini()
406 gk104_fifo_engidx(struct gk104_fifo_priv *priv, u32 engn)  in gk104_fifo_engidx()  argument
425 gk104_fifo_engine(struct gk104_fifo_priv *priv, u32 engn)  in gk104_fifo_engine()  argument
429 	return nvkm_engine(priv, fifo_engine[engn].subdev);  in gk104_fifo_engine()
435 	struct gk104_fifo_priv *priv = container_of(work, typeof(*priv), fault);  in gk104_fifo_recover_work()  local
441 	spin_lock_irqsave(&priv->base.lock, flags);  in gk104_fifo_recover_work()
442 	mask = priv->mask;  in gk104_fifo_recover_work()
443 	priv->mask = 0ULL;  in gk104_fifo_recover_work()
444 	spin_unlock_irqrestore(&priv->base.lock, flags);  in gk104_fifo_recover_work()
447 		engm |= 1 << gk104_fifo_engidx(priv, engn);  in gk104_fifo_recover_work()
448 	nv_mask(priv, 0x002630, engm, engm);  in gk104_fifo_recover_work()
451 		if ((engine = (void *)nvkm_engine(priv, engn))) {  in gk104_fifo_recover_work()
455 		gk104_fifo_runlist_update(priv, gk104_fifo_engidx(priv, engn));  in gk104_fifo_recover_work()
458 	nv_wr32(priv, 0x00262c, engm);  in gk104_fifo_recover_work()
459 	nv_mask(priv, 0x002630, engm, 0x00000000);  in gk104_fifo_recover_work()
463 gk104_fifo_recover(struct gk104_fifo_priv *priv, struct nvkm_engine *engine,  in gk104_fifo_recover()  argument
469 	nv_error(priv, "%s engine fault on channel %d, recovering...\n",  in gk104_fifo_recover()
472 	nv_mask(priv, 0x800004 + (chid * 0x08), 0x00000800, 0x00000800);  in gk104_fifo_recover()
475 	spin_lock_irqsave(&priv->base.lock, flags);  in gk104_fifo_recover()
476 	priv->mask |= 1ULL << nv_engidx(engine);  in gk104_fifo_recover()
477 	spin_unlock_irqrestore(&priv->base.lock, flags);  in gk104_fifo_recover()
478 	schedule_work(&priv->fault);  in gk104_fifo_recover()
482 gk104_fifo_swmthd(struct gk104_fifo_priv *priv, u32 chid, u32 mthd, u32 data)  in gk104_fifo_swmthd()  argument
489 	spin_lock_irqsave(&priv->base.lock, flags);  in gk104_fifo_swmthd()
490 	if (likely(chid >= priv->base.min && chid <= priv->base.max))  in gk104_fifo_swmthd()
491 		chan = (void *)priv->base.channel[chid];  in gk104_fifo_swmthd()
503 	spin_unlock_irqrestore(&priv->base.lock, flags);  in gk104_fifo_swmthd()
519 gk104_fifo_intr_bind(struct gk104_fifo_priv *priv)  in gk104_fifo_intr_bind()  argument
521 	u32 intr = nv_rd32(priv, 0x00252c);  in gk104_fifo_intr_bind()
530 	nv_error(priv, "BIND_ERROR [ %s ]\n", en ? en->name : enunk);  in gk104_fifo_intr_bind()
540 gk104_fifo_intr_sched_ctxsw(struct gk104_fifo_priv *priv)  in gk104_fifo_intr_sched_ctxsw()  argument
547 		u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04));  in gk104_fifo_intr_sched_ctxsw()
558 			if (!(chan = (void *)priv->base.channel[chid]))  in gk104_fifo_intr_sched_ctxsw()
560 			if (!(engine = gk104_fifo_engine(priv, engn)))  in gk104_fifo_intr_sched_ctxsw()
562 			gk104_fifo_recover(priv, engine, chan);  in gk104_fifo_intr_sched_ctxsw()
568 gk104_fifo_intr_sched(struct gk104_fifo_priv *priv)  in gk104_fifo_intr_sched()  argument
570 	u32 intr = nv_rd32(priv, 0x00254c);  in gk104_fifo_intr_sched()
579 	nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk);  in gk104_fifo_intr_sched()
583 		gk104_fifo_intr_sched_ctxsw(priv);  in gk104_fifo_intr_sched()
591 gk104_fifo_intr_chsw(struct gk104_fifo_priv *priv)  in gk104_fifo_intr_chsw()  argument
593 	u32 stat = nv_rd32(priv, 0x00256c);  in gk104_fifo_intr_chsw()
594 	nv_error(priv, "CHSW_ERROR 0x%08x\n", stat);  in gk104_fifo_intr_chsw()
595 	nv_wr32(priv, 0x00256c, stat);  in gk104_fifo_intr_chsw()
599 gk104_fifo_intr_dropped_fault(struct gk104_fifo_priv *priv)  in gk104_fifo_intr_dropped_fault()  argument
601 	u32 stat = nv_rd32(priv, 0x00259c);  in gk104_fifo_intr_dropped_fault()
602 	nv_error(priv, "DROPPED_MMU_FAULT 0x%08x\n", stat);  in gk104_fifo_intr_dropped_fault()
711 gk104_fifo_intr_fault(struct gk104_fifo_priv *priv, int unit)  in gk104_fifo_intr_fault()  argument
713 	u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10));  in gk104_fifo_intr_fault()
714 	u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10));  in gk104_fifo_intr_fault()
715 	u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10));  in gk104_fifo_intr_fault()
716 	u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10));  in gk104_fifo_intr_fault()
738 			nv_mask(priv, 0x001704, 0x00000000, 0x00000000);  in gk104_fifo_intr_fault()
741 			nv_mask(priv, 0x001714, 0x00000000, 0x00000000);  in gk104_fifo_intr_fault()
744 			nv_mask(priv, 0x001718, 0x00000000, 0x00000000);  in gk104_fifo_intr_fault()
747 			engine = nvkm_engine(priv, eu->data2);  in gk104_fifo_intr_fault()
766 	nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on "  in gk104_fifo_intr_fault()
778 			gk104_fifo_recover(priv, engine, (void *)object);  in gk104_fifo_intr_fault()
822 gk104_fifo_intr_pbdma_0(struct gk104_fifo_priv *priv, int unit)  in gk104_fifo_intr_pbdma_0()  argument
824 	u32 mask = nv_rd32(priv, 0x04010c + (unit * 0x2000));  in gk104_fifo_intr_pbdma_0()
825 	u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000)) & mask;  in gk104_fifo_intr_pbdma_0()
826 	u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000));  in gk104_fifo_intr_pbdma_0()
827 	u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000));  in gk104_fifo_intr_pbdma_0()
828 	u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff;  in gk104_fifo_intr_pbdma_0()
834 		if (!gk104_fifo_swmthd(priv, chid, mthd, data))  in gk104_fifo_intr_pbdma_0()
836 		nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008);  in gk104_fifo_intr_pbdma_0()
840 		nv_error(priv, "PBDMA%d:", unit);  in gk104_fifo_intr_pbdma_0()
843 		nv_error(priv,  in gk104_fifo_intr_pbdma_0()
846 			 nvkm_client_name_for_fifo_chid(&priv->base, chid),  in gk104_fifo_intr_pbdma_0()
850 	nv_wr32(priv, 0x040108 + (unit * 0x2000), stat);  in gk104_fifo_intr_pbdma_0()
863 gk104_fifo_intr_pbdma_1(struct gk104_fifo_priv *priv, int unit)  in gk104_fifo_intr_pbdma_1()  argument
865 	u32 mask = nv_rd32(priv, 0x04014c + (unit * 0x2000));  in gk104_fifo_intr_pbdma_1()
866 	u32 stat = nv_rd32(priv, 0x040148 + (unit * 0x2000)) & mask;  in gk104_fifo_intr_pbdma_1()
867 	u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff;  in gk104_fifo_intr_pbdma_1()
870 		nv_error(priv, "PBDMA%d:", unit);  in gk104_fifo_intr_pbdma_1()
873 		nv_error(priv, "PBDMA%d: ch %d %08x %08x\n", unit, chid,  in gk104_fifo_intr_pbdma_1()
874 			 nv_rd32(priv, 0x040150 + (unit * 0x2000)),  in gk104_fifo_intr_pbdma_1()
875 			 nv_rd32(priv, 0x040154 + (unit * 0x2000)));  in gk104_fifo_intr_pbdma_1()
878 	nv_wr32(priv, 0x040148 + (unit * 0x2000), stat);  in gk104_fifo_intr_pbdma_1()
882 gk104_fifo_intr_runlist(struct gk104_fifo_priv *priv)  in gk104_fifo_intr_runlist()  argument
884 	u32 mask = nv_rd32(priv, 0x002a00);  in gk104_fifo_intr_runlist()
887 		wake_up(&priv->engine[engn].wait);  in gk104_fifo_intr_runlist()
888 		nv_wr32(priv, 0x002a00, 1 << engn);  in gk104_fifo_intr_runlist()
894 gk104_fifo_intr_engine(struct gk104_fifo_priv *priv)  in gk104_fifo_intr_engine()  argument
896 	nvkm_fifo_uevent(&priv->base);  in gk104_fifo_intr_engine()
902 	struct gk104_fifo_priv *priv = (void *)subdev;  in gk104_fifo_intr()  local
903 	u32 mask = nv_rd32(priv, 0x002140);  in gk104_fifo_intr()
904 	u32 stat = nv_rd32(priv, 0x002100) & mask;  in gk104_fifo_intr()
907 		gk104_fifo_intr_bind(priv);  in gk104_fifo_intr()
908 		nv_wr32(priv, 0x002100, 0x00000001);  in gk104_fifo_intr()
913 		nv_error(priv, "PIO_ERROR\n");  in gk104_fifo_intr()
914 		nv_wr32(priv, 0x002100, 0x00000010);  in gk104_fifo_intr()
919 		gk104_fifo_intr_sched(priv);  in gk104_fifo_intr()
920 		nv_wr32(priv, 0x002100, 0x00000100);  in gk104_fifo_intr()
925 		gk104_fifo_intr_chsw(priv);  in gk104_fifo_intr()
926 		nv_wr32(priv, 0x002100, 0x00010000);  in gk104_fifo_intr()
931 		nv_error(priv, "FB_FLUSH_TIMEOUT\n");  in gk104_fifo_intr()
932 		nv_wr32(priv, 0x002100, 0x00800000);  in gk104_fifo_intr()
937 		nv_error(priv, "LB_ERROR\n");  in gk104_fifo_intr()
938 		nv_wr32(priv, 0x002100, 0x01000000);  in gk104_fifo_intr()
943 		gk104_fifo_intr_dropped_fault(priv);  in gk104_fifo_intr()
944 		nv_wr32(priv, 0x002100, 0x08000000);  in gk104_fifo_intr()
949 		u32 mask = nv_rd32(priv, 0x00259c);  in gk104_fifo_intr()
952 			gk104_fifo_intr_fault(priv, unit);  in gk104_fifo_intr()
953 			nv_wr32(priv, 0x00259c, (1 << unit));  in gk104_fifo_intr()
960 		u32 mask = nv_rd32(priv, 0x0025a0);  in gk104_fifo_intr()
963 			gk104_fifo_intr_pbdma_0(priv, unit);  in gk104_fifo_intr()
964 			gk104_fifo_intr_pbdma_1(priv, unit);  in gk104_fifo_intr()
965 			nv_wr32(priv, 0x0025a0, (1 << unit));  in gk104_fifo_intr()
972 		gk104_fifo_intr_runlist(priv);  in gk104_fifo_intr()
977 		nv_wr32(priv, 0x002100, 0x80000000);  in gk104_fifo_intr()
978 		gk104_fifo_intr_engine(priv);  in gk104_fifo_intr()
983 		nv_error(priv, "INTR 0x%08x\n", stat);  in gk104_fifo_intr()
984 		nv_mask(priv, 0x002140, stat, 0x00000000);  in gk104_fifo_intr()
985 		nv_wr32(priv, 0x002100, stat);  in gk104_fifo_intr()
1013 	struct gk104_fifo_priv *priv = (void *)object;  in gk104_fifo_fini()  local
1016 	ret = nvkm_fifo_fini(&priv->base, suspend);  in gk104_fifo_fini()
1021 	nv_mask(priv, 0x002140, 0x10000000, 0x10000000);  in gk104_fifo_fini()
1028 	struct gk104_fifo_priv *priv = (void *)object;  in gk104_fifo_init()  local
1031 	ret = nvkm_fifo_init(&priv->base);  in gk104_fifo_init()
1036 	nv_wr32(priv, 0x000204, 0xffffffff);  in gk104_fifo_init()
1037 	priv->spoon_nr = hweight32(nv_rd32(priv, 0x000204));  in gk104_fifo_init()
1038 	nv_debug(priv, "%d PBDMA unit(s)\n", priv->spoon_nr);  in gk104_fifo_init()
1041 	for (i = 0; i < priv->spoon_nr; i++) {  in gk104_fifo_init()
1042 		nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);  in gk104_fifo_init()
1043 		nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */  in gk104_fifo_init()
1044 		nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */  in gk104_fifo_init()
1048 	for (i = 0; i < priv->spoon_nr; i++) {  in gk104_fifo_init()
1049 		nv_wr32(priv, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */  in gk104_fifo_init()
1050 		nv_wr32(priv, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */  in gk104_fifo_init()
1053 	nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12);  in gk104_fifo_init()
1055 	nv_wr32(priv, 0x002100, 0xffffffff);  in gk104_fifo_init()
1056 	nv_wr32(priv, 0x002140, 0x7fffffff);  in gk104_fifo_init()
1063 	struct gk104_fifo_priv *priv = (void *)object;  in gk104_fifo_dtor()  local
1066 	nvkm_gpuobj_unmap(&priv->user.bar);  in gk104_fifo_dtor()
1067 	nvkm_gpuobj_ref(NULL, &priv->user.mem);  in gk104_fifo_dtor()
1070 		nvkm_gpuobj_ref(NULL, &priv->engine[i].runlist[1]);  in gk104_fifo_dtor()
1071 		nvkm_gpuobj_ref(NULL, &priv->engine[i].runlist[0]);  in gk104_fifo_dtor()
1074 	nvkm_fifo_destroy(&priv->base);  in gk104_fifo_dtor()
1083 	struct gk104_fifo_priv *priv;  in gk104_fifo_ctor()  local
1087 			       impl->channels - 1, &priv);  in gk104_fifo_ctor()
1088 	*pobject = nv_object(priv);  in gk104_fifo_ctor()
1092 	INIT_WORK(&priv->fault, gk104_fifo_recover_work);  in gk104_fifo_ctor()
1095 		ret = nvkm_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000,  in gk104_fifo_ctor()
1096 				      0, &priv->engine[i].runlist[0]);  in gk104_fifo_ctor()
1100 		ret = nvkm_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000,  in gk104_fifo_ctor()
1101 				      0, &priv->engine[i].runlist[1]);  in gk104_fifo_ctor()
1105 		init_waitqueue_head(&priv->engine[i].wait);  in gk104_fifo_ctor()
1108 	ret = nvkm_gpuobj_new(nv_object(priv), NULL, impl->channels * 0x200,  in gk104_fifo_ctor()
1109 			      0x1000, NVOBJ_FLAG_ZERO_ALLOC, &priv->user.mem);  in gk104_fifo_ctor()
1113 	ret = nvkm_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW,  in gk104_fifo_ctor()
1114 			      &priv->user.bar);  in gk104_fifo_ctor()
1118 	ret = nvkm_event_init(&gk104_fifo_uevent_func, 1, 1, &priv->base.uevent);  in gk104_fifo_ctor()
1122 	nv_subdev(priv)->unit = 0x00000100;  in gk104_fifo_ctor()
1123 	nv_subdev(priv)->intr = gk104_fifo_intr;  in gk104_fifo_ctor()
1124 	nv_engine(priv)->cclass = &gk104_fifo_cclass;  in gk104_fifo_ctor()
1125 	nv_engine(priv)->sclass = gk104_fifo_sclass;  in gk104_fifo_ctor()