root/drivers/infiniband/core/uverbs_ioctl.c

DEFINITIONS

1. uapi_compute_bundle_size
2. _uverbs_alloc
3. uverbs_is_attr_cleared
4. uverbs_set_output
5. uverbs_process_idrs_array
6. uverbs_free_idrs_array
7. uverbs_process_attr
8. uapi_get_attr_for_method
9. uverbs_set_attr
10. ib_uverbs_run_method
11. bundle_destroy
12. ib_uverbs_cmd_verbs
13. ib_uverbs_ioctl
14. uverbs_get_flags64
15. uverbs_get_flags32
16. uverbs_fill_udata
17. uverbs_copy_to
18. uverbs_output_written
19. _uverbs_get_const
20. uverbs_copy_to_struct_or_zero

   1 /*
   2  * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 
  33 #include <rdma/rdma_user_ioctl.h>
  34 #include <rdma/uverbs_ioctl.h>
  35 #include "rdma_core.h"
  36 #include "uverbs.h"
  37 
  38 struct bundle_alloc_head {
  39         struct bundle_alloc_head *next;
  40         u8 data[];
  41 };
  42 
  43 struct bundle_priv {
  44         /* Must be first */
  45         struct bundle_alloc_head alloc_head;
  46         struct bundle_alloc_head *allocated_mem;
  47         size_t internal_avail;
  48         size_t internal_used;
  49 
  50         struct radix_tree_root *radix;
  51         const struct uverbs_api_ioctl_method *method_elm;
  52         void __rcu **radix_slots;
  53         unsigned long radix_slots_len;
  54         u32 method_key;
  55 
  56         struct ib_uverbs_attr __user *user_attrs;
  57         struct ib_uverbs_attr *uattrs;
  58 
  59         DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
  60         DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
  61 
  62         /*
  63          * Must be last. bundle ends in a flex array which overlaps
  64          * internal_buffer.
  65          */
  66         struct uverbs_attr_bundle bundle;
  67         u64 internal_buffer[32];
  68 };
  69 
  70 /*
  71  * Each method has an absolute minimum amount of memory it needs to allocate,
  72  * precompute that amount and determine if the onstack memory can be used or
  73  * if allocation is need.
  74  */
  75 void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
  76                               unsigned int num_attrs)
  77 {
  78         struct bundle_priv *pbundle;
  79         size_t bundle_size =
  80                 offsetof(struct bundle_priv, internal_buffer) +
  81                 sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len +
  82                 sizeof(*pbundle->uattrs) * num_attrs;
  83 
  84         method_elm->use_stack = bundle_size <= sizeof(*pbundle);
  85         method_elm->bundle_size =
  86                 ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));
  87 
  88         /* Do not want order-2 allocations for this. */
  89         WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
  90 }
  91 
  92 /**
  93  * uverbs_alloc() - Quickly allocate memory for use with a bundle
  94  * @bundle: The bundle
  95  * @size: Number of bytes to allocate
  96  * @flags: Allocator flags
  97  *
  98  * The bundle allocator is intended for allocations that are connected with
  99  * processing the system call related to the bundle. The allocated memory is
 100  * always freed once the system call completes, and cannot be freed any other
 101  * way.
 102  *
 103  * This tries to use a small pool of pre-allocated memory for performance.
 104  */
 105 __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
 106                              gfp_t flags)
 107 {
 108         struct bundle_priv *pbundle =
 109                 container_of(bundle, struct bundle_priv, bundle);
 110         size_t new_used;
 111         void *res;
 112 
 113         if (check_add_overflow(size, pbundle->internal_used, &new_used))
 114                 return ERR_PTR(-EOVERFLOW);
 115 
 116         if (new_used > pbundle->internal_avail) {
 117                 struct bundle_alloc_head *buf;
 118 
 119                 buf = kvmalloc(struct_size(buf, data, size), flags);
 120                 if (!buf)
 121                         return ERR_PTR(-ENOMEM);
 122                 buf->next = pbundle->allocated_mem;
 123                 pbundle->allocated_mem = buf;
 124                 return buf->data;
 125         }
 126 
 127         res = (void *)pbundle->internal_buffer + pbundle->internal_used;
 128         pbundle->internal_used =
 129                 ALIGN(new_used, sizeof(*pbundle->internal_buffer));
 130         if (want_init_on_alloc(flags))
 131                 memset(res, 0, size);
 132         return res;
 133 }
 134 EXPORT_SYMBOL(_uverbs_alloc);
 135 
 136 static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
 137                                    u16 len)
 138 {
 139         if (uattr->len > sizeof(((struct ib_uverbs_attr *)0)->data))
 140                 return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
 141                                             uattr->len - len);
 142 
 143         return !memchr_inv((const void *)&uattr->data + len,
 144                            0, uattr->len - len);
 145 }
 146 
 147 static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
 148                              const struct uverbs_attr *attr)
 149 {
 150         struct bundle_priv *pbundle =
 151                 container_of(bundle, struct bundle_priv, bundle);
 152         u16 flags;
 153 
 154         flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
 155                 UVERBS_ATTR_F_VALID_OUTPUT;
 156         if (put_user(flags,
 157                      &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
 158                 return -EFAULT;
 159         return 0;
 160 }
 161 
 162 static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
 163                                      const struct uverbs_api_attr *attr_uapi,
 164                                      struct uverbs_objs_arr_attr *attr,
 165                                      struct ib_uverbs_attr *uattr,
 166                                      u32 attr_bkey)
 167 {
 168         const struct uverbs_attr_spec *spec = &attr_uapi->spec;
 169         size_t array_len;
 170         u32 *idr_vals;
 171         int ret = 0;
 172         size_t i;
 173 
 174         if (uattr->attr_data.reserved)
 175                 return -EINVAL;
 176 
 177         if (uattr->len % sizeof(u32))
 178                 return -EINVAL;
 179 
 180         array_len = uattr->len / sizeof(u32);
 181         if (array_len < spec->u2.objs_arr.min_len ||
 182             array_len > spec->u2.objs_arr.max_len)
 183                 return -EINVAL;
 184 
 185         attr->uobjects =
 186                 uverbs_alloc(&pbundle->bundle,
 187                              array_size(array_len, sizeof(*attr->uobjects)));
 188         if (IS_ERR(attr->uobjects))
 189                 return PTR_ERR(attr->uobjects);
 190 
 191         /*
 192          * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects
 193          * to store idrs array and avoid additional memory allocation. The
 194          * idrs array is offset to the end of the uobjects array so we will be
 195          * able to read idr and replace with a pointer.
 196          */
 197         idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;
 198 
 199         if (uattr->len > sizeof(uattr->data)) {
 200                 ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data),
 201                                      uattr->len);
 202                 if (ret)
 203                         return -EFAULT;
 204         } else {
 205                 memcpy(idr_vals, &uattr->data, uattr->len);
 206         }
 207 
 208         for (i = 0; i != array_len; i++) {
 209                 attr->uobjects[i] = uverbs_get_uobject_from_file(
 210                         spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,
 211                         idr_vals[i], &pbundle->bundle);
 212                 if (IS_ERR(attr->uobjects[i])) {
 213                         ret = PTR_ERR(attr->uobjects[i]);
 214                         break;
 215                 }
 216         }
 217 
 218         attr->len = i;
 219         __set_bit(attr_bkey, pbundle->spec_finalize);
 220         return ret;
 221 }
 222 
 223 static int uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
 224                                   struct uverbs_objs_arr_attr *attr,
 225                                   bool commit, struct uverbs_attr_bundle *attrs)
 226 {
 227         const struct uverbs_attr_spec *spec = &attr_uapi->spec;
 228         int current_ret;
 229         int ret = 0;
 230         size_t i;
 231 
 232         for (i = 0; i != attr->len; i++) {
 233                 current_ret = uverbs_finalize_object(attr->uobjects[i],
 234                                                      spec->u2.objs_arr.access,
 235                                                      commit, attrs);
 236                 if (!ret)
 237                         ret = current_ret;
 238         }
 239 
 240         return ret;
 241 }
 242 
 243 static int uverbs_process_attr(struct bundle_priv *pbundle,
 244                                const struct uverbs_api_attr *attr_uapi,
 245                                struct ib_uverbs_attr *uattr, u32 attr_bkey)
 246 {
 247         const struct uverbs_attr_spec *spec = &attr_uapi->spec;
 248         struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey];
 249         const struct uverbs_attr_spec *val_spec = spec;
 250         struct uverbs_obj_attr *o_attr;
 251 
 252         switch (spec->type) {
 253         case UVERBS_ATTR_TYPE_ENUM_IN:
 254                 if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
 255                         return -EOPNOTSUPP;
 256 
 257                 if (uattr->attr_data.enum_data.reserved)
 258                         return -EINVAL;
 259 
 260                 val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];
 261 
 262                 /* Currently we only support PTR_IN based enums */
 263                 if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
 264                         return -EOPNOTSUPP;
 265 
 266                 e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
 267         /* fall through */
 268         case UVERBS_ATTR_TYPE_PTR_IN:
 269                 /* Ensure that any data provided by userspace beyond the known
 270                  * struct is zero. Userspace that knows how to use some future
 271                  * longer struct will fail here if used with an old kernel and
 272                  * non-zero content, making ABI compat/discovery simpler.
 273                  */
 274                 if (uattr->len > val_spec->u.ptr.len &&
 275                     val_spec->zero_trailing &&
 276                     !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
 277                         return -EOPNOTSUPP;
 278 
 279         /* fall through */
 280         case UVERBS_ATTR_TYPE_PTR_OUT:
 281                 if (uattr->len < val_spec->u.ptr.min_len ||
 282                     (!val_spec->zero_trailing &&
 283                      uattr->len > val_spec->u.ptr.len))
 284                         return -EINVAL;
 285 
 286                 if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
 287                     uattr->attr_data.reserved)
 288                         return -EINVAL;
 289 
 290                 e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
 291                 e->ptr_attr.len = uattr->len;
 292 
 293                 if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
 294                         void *p;
 295 
 296                         p = uverbs_alloc(&pbundle->bundle, uattr->len);
 297                         if (IS_ERR(p))
 298                                 return PTR_ERR(p);
 299 
 300                         e->ptr_attr.ptr = p;
 301 
 302                         if (copy_from_user(p, u64_to_user_ptr(uattr->data),
 303                                            uattr->len))
 304                                 return -EFAULT;
 305                 } else {
 306                         e->ptr_attr.data = uattr->data;
 307                 }
 308                 break;
 309 
 310         case UVERBS_ATTR_TYPE_IDR:
 311         case UVERBS_ATTR_TYPE_FD:
 312                 if (uattr->attr_data.reserved)
 313                         return -EINVAL;
 314 
 315                 if (uattr->len != 0)
 316                         return -EINVAL;
 317 
 318                 o_attr = &e->obj_attr;
 319                 o_attr->attr_elm = attr_uapi;
 320 
 321                 /*
 322                  * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
 323                  * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
 324                  * here without caring about truncation as we know that the
 325                  * IDR implementation today rejects negative IDs
 326                  */
 327                 o_attr->uobject = uverbs_get_uobject_from_file(
 328                         spec->u.obj.obj_type, spec->u.obj.access,
 329                         uattr->data_s64, &pbundle->bundle);
 330                 if (IS_ERR(o_attr->uobject))
 331                         return PTR_ERR(o_attr->uobject);
 332                 __set_bit(attr_bkey, pbundle->uobj_finalize);
 333 
 334                 if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
 335                         unsigned int uattr_idx = uattr - pbundle->uattrs;
 336                         s64 id = o_attr->uobject->id;
 337 
 338                         /* Copy the allocated id to the user-space */
 339                         if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
 340                                 return -EFAULT;
 341                 }
 342 
 343                 break;
 344 
 345         case UVERBS_ATTR_TYPE_IDRS_ARRAY:
 346                 return uverbs_process_idrs_array(pbundle, attr_uapi,
 347                                                  &e->objs_arr_attr, uattr,
 348                                                  attr_bkey);
 349         default:
 350                 return -EOPNOTSUPP;
 351         }
 352 
 353         return 0;
 354 }
 355 
 356 /*
 357  * We search the radix tree with the method prefix and now we want to fast
 358  * search the suffix bits to get a particular attribute pointer. It is not
 359  * totally clear to me if this breaks the radix tree encasulation or not, but
 360  * it uses the iter data to determine if the method iter points at the same
 361  * chunk that will store the attribute, if so it just derefs it directly. By
 362  * construction in most kernel configs the method and attrs will all fit in a
 363  * single radix chunk, so in most cases this will have no search. Other cases
 364  * this falls back to a full search.
 365  */
 366 static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
 367                                              u32 attr_key)
 368 {
 369         void __rcu **slot;
 370 
 371         if (likely(attr_key < pbundle->radix_slots_len)) {
 372                 void *entry;
 373 
 374                 slot = pbundle->radix_slots + attr_key;
 375                 entry = rcu_dereference_raw(*slot);
 376                 if (likely(!radix_tree_is_internal_node(entry) && entry))
 377                         return slot;
 378         }
 379 
 380         return radix_tree_lookup_slot(pbundle->radix,
 381                                       pbundle->method_key | attr_key);
 382 }
 383 
 384 static int uverbs_set_attr(struct bundle_priv *pbundle,
 385                            struct ib_uverbs_attr *uattr)
 386 {
 387         u32 attr_key = uapi_key_attr(uattr->attr_id);
 388         u32 attr_bkey = uapi_bkey_attr(attr_key);
 389         const struct uverbs_api_attr *attr;
 390         void __rcu **slot;
 391         int ret;
 392 
 393         slot = uapi_get_attr_for_method(pbundle, attr_key);
 394         if (!slot) {
 395                 /*
 396                  * Kernel does not support the attribute but user-space says it
 397                  * is mandatory
 398                  */
 399                 if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
 400                         return -EPROTONOSUPPORT;
 401                 return 0;
 402         }
 403         attr = rcu_dereference_protected(*slot, true);
 404 
 405         /* Reject duplicate attributes from user-space */
 406         if (test_bit(attr_bkey, pbundle->bundle.attr_present))
 407                 return -EINVAL;
 408 
 409         ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
 410         if (ret)
 411                 return ret;
 412 
 413         __set_bit(attr_bkey, pbundle->bundle.attr_present);
 414 
 415         return 0;
 416 }
 417 
 418 static int ib_uverbs_run_method(struct bundle_priv *pbundle,
 419                                 unsigned int num_attrs)
 420 {
 421         int (*handler)(struct uverbs_attr_bundle *attrs);
 422         size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
 423         unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
 424         unsigned int i;
 425         int ret;
 426 
 427         /* See uverbs_disassociate_api() */
 428         handler = srcu_dereference(
 429                 pbundle->method_elm->handler,
 430                 &pbundle->bundle.ufile->device->disassociate_srcu);
 431         if (!handler)
 432                 return -EIO;
 433 
 434         pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size);
 435         if (IS_ERR(pbundle->uattrs))
 436                 return PTR_ERR(pbundle->uattrs);
 437         if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
 438                 return -EFAULT;
 439 
 440         for (i = 0; i != num_attrs; i++) {
 441                 ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
 442                 if (unlikely(ret))
 443                         return ret;
 444         }
 445 
 446         /* User space did not provide all the mandatory attributes */
 447         if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
 448                                     pbundle->bundle.attr_present,
 449                                     pbundle->method_elm->key_bitmap_len)))
 450                 return -EINVAL;
 451 
 452         if (pbundle->method_elm->has_udata)
 453                 uverbs_fill_udata(&pbundle->bundle,
 454                                   &pbundle->bundle.driver_udata,
 455                                   UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
 456         else
 457                 pbundle->bundle.driver_udata = (struct ib_udata){};
 458 
 459         if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
 460                 struct uverbs_obj_attr *destroy_attr =
 461                         &pbundle->bundle.attrs[destroy_bkey].obj_attr;
 462 
 463                 ret = uobj_destroy(destroy_attr->uobject, &pbundle->bundle);
 464                 if (ret)
 465                         return ret;
 466                 __clear_bit(destroy_bkey, pbundle->uobj_finalize);
 467 
 468                 ret = handler(&pbundle->bundle);
 469                 uobj_put_destroy(destroy_attr->uobject);
 470         } else {
 471                 ret = handler(&pbundle->bundle);
 472         }
 473 
 474         /*
 475          * Until the drivers are revised to use the bundle directly we have to
 476          * assume that the driver wrote to its UHW_OUT and flag userspace
 477          * appropriately.
 478          */
 479         if (!ret && pbundle->method_elm->has_udata) {
 480                 const struct uverbs_attr *attr =
 481                         uverbs_attr_get(&pbundle->bundle, UVERBS_ATTR_UHW_OUT);
 482 
 483                 if (!IS_ERR(attr))
 484                         ret = uverbs_set_output(&pbundle->bundle, attr);
 485         }
 486 
 487         /*
 488          * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
 489          * not invoke the method because the request is not supported.  No
 490          * other cases should return this code.
 491          */
 492         if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
 493                 return -EINVAL;
 494 
 495         return ret;
 496 }
 497 
 498 static int bundle_destroy(struct bundle_priv *pbundle, bool commit)
 499 {
 500         unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
 501         struct bundle_alloc_head *memblock;
 502         unsigned int i;
 503         int ret = 0;
 504 
 505         /* fast path for simple uobjects */
 506         i = -1;
 507         while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
 508                                   i + 1)) < key_bitmap_len) {
 509                 struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
 510                 int current_ret;
 511 
 512                 current_ret = uverbs_finalize_object(
 513                         attr->obj_attr.uobject,
 514                         attr->obj_attr.attr_elm->spec.u.obj.access, commit,
 515                         &pbundle->bundle);
 516                 if (!ret)
 517                         ret = current_ret;
 518         }
 519 
 520         i = -1;
 521         while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len,
 522                                   i + 1)) < key_bitmap_len) {
 523                 struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
 524                 const struct uverbs_api_attr *attr_uapi;
 525                 void __rcu **slot;
 526                 int current_ret;
 527 
 528                 slot = uapi_get_attr_for_method(
 529                         pbundle,
 530                         pbundle->method_key | uapi_bkey_to_key_attr(i));
 531                 if (WARN_ON(!slot))
 532                         continue;
 533 
 534                 attr_uapi = rcu_dereference_protected(*slot, true);
 535 
 536                 if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
 537                         current_ret = uverbs_free_idrs_array(
 538                                 attr_uapi, &attr->objs_arr_attr, commit,
 539                                 &pbundle->bundle);
 540                         if (!ret)
 541                                 ret = current_ret;
 542                 }
 543         }
 544 
 545         for (memblock = pbundle->allocated_mem; memblock;) {
 546                 struct bundle_alloc_head *tmp = memblock;
 547 
 548                 memblock = memblock->next;
 549                 kvfree(tmp);
 550         }
 551 
 552         return ret;
 553 }
 554 
 555 static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
 556                                struct ib_uverbs_ioctl_hdr *hdr,
 557                                struct ib_uverbs_attr __user *user_attrs)
 558 {
 559         const struct uverbs_api_ioctl_method *method_elm;
 560         struct uverbs_api *uapi = ufile->device->uapi;
 561         struct radix_tree_iter attrs_iter;
 562         struct bundle_priv *pbundle;
 563         struct bundle_priv onstack;
 564         void __rcu **slot;
 565         int destroy_ret;
 566         int ret;
 567 
 568         if (unlikely(hdr->driver_id != uapi->driver_id))
 569                 return -EINVAL;
 570 
 571         slot = radix_tree_iter_lookup(
 572                 &uapi->radix, &attrs_iter,
 573                 uapi_key_obj(hdr->object_id) |
 574                         uapi_key_ioctl_method(hdr->method_id));
 575         if (unlikely(!slot))
 576                 return -EPROTONOSUPPORT;
 577         method_elm = rcu_dereference_protected(*slot, true);
 578 
 579         if (!method_elm->use_stack) {
 580                 pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
 581                 if (!pbundle)
 582                         return -ENOMEM;
 583                 pbundle->internal_avail =
 584                         method_elm->bundle_size -
 585                         offsetof(struct bundle_priv, internal_buffer);
 586                 pbundle->alloc_head.next = NULL;
 587                 pbundle->allocated_mem = &pbundle->alloc_head;
 588         } else {
 589                 pbundle = &onstack;
 590                 pbundle->internal_avail = sizeof(pbundle->internal_buffer);
 591                 pbundle->allocated_mem = NULL;
 592         }
 593 
 594         /* Space for the pbundle->bundle.attrs flex array */
 595         pbundle->method_elm = method_elm;
 596         pbundle->method_key = attrs_iter.index;
 597         pbundle->bundle.ufile = ufile;
 598         pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
 599         pbundle->radix = &uapi->radix;
 600         pbundle->radix_slots = slot;
 601         pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
 602         pbundle->user_attrs = user_attrs;
 603 
 604         pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
 605                                                sizeof(*pbundle->bundle.attrs),
 606                                        sizeof(*pbundle->internal_buffer));
 607         memset(pbundle->bundle.attr_present, 0,
 608                sizeof(pbundle->bundle.attr_present));
 609         memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
 610         memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
 611 
 612         ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
 613         destroy_ret = bundle_destroy(pbundle, ret == 0);
 614         if (unlikely(destroy_ret && !ret))
 615                 return destroy_ret;
 616 
 617         return ret;
 618 }
 619 
 620 long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 621 {
 622         struct ib_uverbs_file *file = filp->private_data;
 623         struct ib_uverbs_ioctl_hdr __user *user_hdr =
 624                 (struct ib_uverbs_ioctl_hdr __user *)arg;
 625         struct ib_uverbs_ioctl_hdr hdr;
 626         int srcu_key;
 627         int err;
 628 
 629         if (unlikely(cmd != RDMA_VERBS_IOCTL))
 630                 return -ENOIOCTLCMD;
 631 
 632         err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
 633         if (err)
 634                 return -EFAULT;
 635 
 636         if (hdr.length > PAGE_SIZE ||
 637             hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
 638                 return -EINVAL;
 639 
 640         if (hdr.reserved1 || hdr.reserved2)
 641                 return -EPROTONOSUPPORT;
 642 
 643         srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
 644         err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
 645         srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
 646         return err;
 647 }
 648 
 649 int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
 650                        size_t idx, u64 allowed_bits)
 651 {
 652         const struct uverbs_attr *attr;
 653         u64 flags;
 654 
 655         attr = uverbs_attr_get(attrs_bundle, idx);
 656         /* Missing attribute means 0 flags */
 657         if (IS_ERR(attr)) {
 658                 *to = 0;
 659                 return 0;
 660         }
 661 
 662         /*
 663          * New userspace code should use 8 bytes to pass flags, but we
 664          * transparently support old userspaces that were using 4 bytes as
 665          * well.
 666          */
 667         if (attr->ptr_attr.len == 8)
 668                 flags = attr->ptr_attr.data;
 669         else if (attr->ptr_attr.len == 4)
 670                 flags = *(u32 *)&attr->ptr_attr.data;
 671         else
 672                 return -EINVAL;
 673 
 674         if (flags & ~allowed_bits)
 675                 return -EINVAL;
 676 
 677         *to = flags;
 678         return 0;
 679 }
 680 EXPORT_SYMBOL(uverbs_get_flags64);
 681 
 682 int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
 683                        size_t idx, u64 allowed_bits)
 684 {
 685         u64 flags;
 686         int ret;
 687 
 688         ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
 689         if (ret)
 690                 return ret;
 691 
 692         if (flags > U32_MAX)
 693                 return -EINVAL;
 694         *to = flags;
 695 
 696         return 0;
 697 }
 698 EXPORT_SYMBOL(uverbs_get_flags32);
 699 
 700 /*
 701  * Fill a ib_udata struct (core or uhw) using the given attribute IDs.
 702  * This is primarily used to convert the UVERBS_ATTR_UHW() into the
 703  * ib_udata format used by the drivers.
 704  */
 705 void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
 706                        struct ib_udata *udata, unsigned int attr_in,
 707                        unsigned int attr_out)
 708 {
 709         struct bundle_priv *pbundle =
 710                 container_of(bundle, struct bundle_priv, bundle);
 711         const struct uverbs_attr *in =
 712                 uverbs_attr_get(&pbundle->bundle, attr_in);
 713         const struct uverbs_attr *out =
 714                 uverbs_attr_get(&pbundle->bundle, attr_out);
 715 
 716         if (!IS_ERR(in)) {
 717                 udata->inlen = in->ptr_attr.len;
 718                 if (uverbs_attr_ptr_is_inline(in))
 719                         udata->inbuf =
 720                                 &pbundle->user_attrs[in->ptr_attr.uattr_idx]
 721                                          .data;
 722                 else
 723                         udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);
 724         } else {
 725                 udata->inbuf = NULL;
 726                 udata->inlen = 0;
 727         }
 728 
 729         if (!IS_ERR(out)) {
 730                 udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);
 731                 udata->outlen = out->ptr_attr.len;
 732         } else {
 733                 udata->outbuf = NULL;
 734                 udata->outlen = 0;
 735         }
 736 }
 737 
 738 int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
 739                    const void *from, size_t size)
 740 {
 741         const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
 742         size_t min_size;
 743 
 744         if (IS_ERR(attr))
 745                 return PTR_ERR(attr);
 746 
 747         min_size = min_t(size_t, attr->ptr_attr.len, size);
 748         if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
 749                 return -EFAULT;
 750 
 751         return uverbs_set_output(bundle, attr);
 752 }
 753 EXPORT_SYMBOL(uverbs_copy_to);
 754 
 755 
 756 /*
 757  * This is only used if the caller has directly used copy_to_use to write the
 758  * data.  It signals to user space that the buffer is filled in.
 759  */
 760 int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
 761 {
 762         const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
 763 
 764         if (IS_ERR(attr))
 765                 return PTR_ERR(attr);
 766 
 767         return uverbs_set_output(bundle, attr);
 768 }
 769 
 770 int _uverbs_get_const(s64 *to, const struct uverbs_attr_bundle *attrs_bundle,
 771                       size_t idx, s64 lower_bound, u64 upper_bound,
 772                       s64  *def_val)
 773 {
 774         const struct uverbs_attr *attr;
 775 
 776         attr = uverbs_attr_get(attrs_bundle, idx);
 777         if (IS_ERR(attr)) {
 778                 if ((PTR_ERR(attr) != -ENOENT) || !def_val)
 779                         return PTR_ERR(attr);
 780 
 781                 *to = *def_val;
 782         } else {
 783                 *to = attr->ptr_attr.data;
 784         }
 785 
 786         if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
 787                 return -EINVAL;
 788 
 789         return 0;
 790 }
 791 EXPORT_SYMBOL(_uverbs_get_const);
 792 
 793 int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
 794                                   size_t idx, const void *from, size_t size)
 795 {
 796         const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
 797 
 798         if (size < attr->ptr_attr.len) {
 799                 if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
 800                                attr->ptr_attr.len - size))
 801                         return -EFAULT;
 802         }
 803         return uverbs_copy_to(bundle, idx, from, size);
 804 }