40 static void __scm_free_rq(struct scm_request *scmrq)  in __scm_free_rq()  argument
42 	struct aob_rq_header *aobrq = to_aobrq(scmrq);  in __scm_free_rq()
44 	free_page((unsigned long) scmrq->aob);  in __scm_free_rq()
45 	__scm_free_rq_cluster(scmrq);  in __scm_free_rq()
46 	kfree(scmrq->request);  in __scm_free_rq()
53 	struct scm_request *scmrq;  in scm_free_rqs()  local
57 		scmrq = list_entry(iter, struct scm_request, list);  in scm_free_rqs()
58 		list_del(&scmrq->list);  in scm_free_rqs()
59 		__scm_free_rq(scmrq);  in scm_free_rqs()
69 	struct scm_request *scmrq;  in __scm_alloc_rq()  local
71 	aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);  in __scm_alloc_rq()
75 	scmrq = (void *) aobrq->data;  in __scm_alloc_rq()
76 	scmrq->aob = (void *) get_zeroed_page(GFP_DMA);  in __scm_alloc_rq()
77 	if (!scmrq->aob)  in __scm_alloc_rq()
80 	scmrq->request = kcalloc(nr_requests_per_io, sizeof(scmrq->request[0]),  in __scm_alloc_rq()
82 	if (!scmrq->request)  in __scm_alloc_rq()
85 	if (__scm_alloc_rq_cluster(scmrq))  in __scm_alloc_rq()
88 	INIT_LIST_HEAD(&scmrq->list);  in __scm_alloc_rq()
90 	list_add(&scmrq->list, &inactive_requests);  in __scm_alloc_rq()
95 	__scm_free_rq(scmrq);  in __scm_alloc_rq()
115 	struct scm_request *scmrq = NULL;  in scm_request_fetch()  local
120 	scmrq = list_first_entry(&inactive_requests, struct scm_request, list);  in scm_request_fetch()
121 	list_del(&scmrq->list);  in scm_request_fetch()
124 	return scmrq;  in scm_request_fetch()
127 static void scm_request_done(struct scm_request *scmrq)  in scm_request_done()  argument
134 	for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {  in scm_request_done()
135 		msb = &scmrq->aob->msb[i];  in scm_request_done()
144 	list_add(&scmrq->list, &inactive_requests);  in scm_request_done()
168 struct aidaw *scm_aidaw_fetch(struct scm_request *scmrq, unsigned int bytes)  in scm_aidaw_fetch()  argument
172 	if (scm_aidaw_bytes(scmrq->next_aidaw) >= bytes)  in scm_aidaw_fetch()
173 		return scmrq->next_aidaw;  in scm_aidaw_fetch()
181 static int scm_request_prepare(struct scm_request *scmrq)  in scm_request_prepare()  argument
183 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_request_prepare()
185 	int pos = scmrq->aob->request.msb_count;  in scm_request_prepare()
186 	struct msb *msb = &scmrq->aob->msb[pos];  in scm_request_prepare()
187 	struct request *req = scmrq->request[pos];  in scm_request_prepare()
192 	aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req));  in scm_request_prepare()
197 	scmrq->aob->request.msb_count++;  in scm_request_prepare()
210 	scmrq->next_aidaw = aidaw;  in scm_request_prepare()
214 static inline void scm_request_set(struct scm_request *scmrq,  in scm_request_set()  argument
217 	scmrq->request[scmrq->aob->request.msb_count] = req;  in scm_request_set()
221 				    struct scm_request *scmrq)  in scm_request_init()  argument
223 	struct aob_rq_header *aobrq = to_aobrq(scmrq);  in scm_request_init()
224 	struct aob *aob = scmrq->aob;  in scm_request_init()
226 	memset(scmrq->request, 0,  in scm_request_init()
227 	       nr_requests_per_io * sizeof(scmrq->request[0]));  in scm_request_init()
232 	scmrq->bdev = bdev;  in scm_request_init()
233 	scmrq->retries = 4;  in scm_request_init()
234 	scmrq->error = 0;  in scm_request_init()
236 	scmrq->next_aidaw = (void *) &aob->msb[nr_requests_per_io];  in scm_request_init()
237 	scm_request_cluster_init(scmrq);  in scm_request_init()
249 void scm_request_requeue(struct scm_request *scmrq)  in scm_request_requeue()  argument
251 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_request_requeue()
254 	scm_release_cluster(scmrq);  in scm_request_requeue()
255 	for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)  in scm_request_requeue()
256 		blk_requeue_request(bdev->rq, scmrq->request[i]);  in scm_request_requeue()
259 	scm_request_done(scmrq);  in scm_request_requeue()
263 void scm_request_finish(struct scm_request *scmrq)  in scm_request_finish()  argument
265 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_request_finish()
268 	scm_release_cluster(scmrq);  in scm_request_finish()
269 	for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)  in scm_request_finish()
270 		blk_end_request_all(scmrq->request[i], scmrq->error);  in scm_request_finish()
273 	scm_request_done(scmrq);  in scm_request_finish()
276 static int scm_request_start(struct scm_request *scmrq)  in scm_request_start()  argument
278 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_request_start()
282 	if (!scmrq->aob->request.msb_count) {  in scm_request_start()
283 		scm_request_requeue(scmrq);  in scm_request_start()
287 	ret = eadm_start_aob(scmrq->aob);  in scm_request_start()
290 		scm_request_requeue(scmrq);  in scm_request_start()
299 	struct scm_request *scmrq = NULL;  in scm_blk_request()  local
313 		if (!scmrq) {  in scm_blk_request()
314 			scmrq = scm_request_fetch();  in scm_blk_request()
315 			if (!scmrq) {  in scm_blk_request()
319 			scm_request_init(bdev, scmrq);  in scm_blk_request()
321 		scm_request_set(scmrq, req);  in scm_blk_request()
323 		if (!scm_reserve_cluster(scmrq)) {  in scm_blk_request()
325 			scm_request_set(scmrq, NULL);  in scm_blk_request()
326 			if (scmrq->aob->request.msb_count)  in scm_blk_request()
329 			scm_request_done(scmrq);  in scm_blk_request()
333 		if (scm_need_cluster_request(scmrq)) {  in scm_blk_request()
334 			if (scmrq->aob->request.msb_count) {  in scm_blk_request()
336 				scm_request_set(scmrq, NULL);  in scm_blk_request()
337 				if (scm_request_start(scmrq))  in scm_blk_request()
342 				scm_initiate_cluster_request(scmrq);  in scm_blk_request()
344 			scmrq = NULL;  in scm_blk_request()
348 		if (scm_request_prepare(scmrq)) {  in scm_blk_request()
350 			scm_request_set(scmrq, NULL);  in scm_blk_request()
355 		if (scmrq->aob->request.msb_count < nr_requests_per_io)  in scm_blk_request()
358 		if (scm_request_start(scmrq))  in scm_blk_request()
361 		scmrq = NULL;  in scm_blk_request()
364 	if (scmrq)  in scm_blk_request()
365 		scm_request_start(scmrq);  in scm_blk_request()
370 static void __scmrq_log_error(struct scm_request *scmrq)  in __scmrq_log_error()  argument
372 	struct aob *aob = scmrq->aob;  in __scmrq_log_error()
374 	if (scmrq->error == -ETIMEDOUT)  in __scmrq_log_error()
380 	if (scmrq->retries)  in __scmrq_log_error()
384 		       scmrq->error);  in __scmrq_log_error()
389 	struct scm_request *scmrq = data;  in scm_blk_irq()  local
390 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_blk_irq()
392 	scmrq->error = error;  in scm_blk_irq()
394 		__scmrq_log_error(scmrq);  in scm_blk_irq()
397 	list_add_tail(&scmrq->list, &bdev->finished_requests);  in scm_blk_irq()
402 static void scm_blk_handle_error(struct scm_request *scmrq)  in scm_blk_handle_error()  argument
404 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_blk_handle_error()
407 	if (scmrq->error != -EIO)  in scm_blk_handle_error()
411 	switch (scmrq->aob->response.eqc) {  in scm_blk_handle_error()
425 	if (!eadm_start_aob(scmrq->aob))  in scm_blk_handle_error()
430 	scm_request_requeue(scmrq);  in scm_blk_handle_error()
436 	struct scm_request *scmrq;  in scm_blk_tasklet()  local
441 		scmrq = list_first_entry(&bdev->finished_requests,  in scm_blk_tasklet()
443 		list_del(&scmrq->list);  in scm_blk_tasklet()
446 		if (scmrq->error && scmrq->retries-- > 0) {  in scm_blk_tasklet()
447 			scm_blk_handle_error(scmrq);  in scm_blk_tasklet()
454 		if (scm_test_cluster_request(scmrq)) {  in scm_blk_tasklet()
455 			scm_cluster_request_irq(scmrq);  in scm_blk_tasklet()
460 		scm_request_finish(scmrq);  in scm_blk_tasklet()