1/* -*- mode: c; c-basic-offset: 8; -*- 2 * 3 * vim: noexpandtab sw=8 ts=8 sts=0: 4 * 5 * Copyright (C) 2004 Oracle. All rights reserved. 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public 9 * License as published by the Free Software Foundation; either 10 * version 2 of the License, or (at your option) any later version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public 18 * License along with this program; if not, write to the 19 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 20 * Boston, MA 021110-1307, USA. 21 * 22 * ---- 23 * 24 * Callers for this were originally written against a very simple synchronus 25 * API. This implementation reflects those simple callers. Some day I'm sure 26 * we'll need to move to a more robust posting/callback mechanism. 27 * 28 * Transmit calls pass in kernel virtual addresses and block copying this into 29 * the socket's tx buffers via a usual blocking sendmsg. They'll block waiting 30 * for a failed socket to timeout. TX callers can also pass in a poniter to an 31 * 'int' which gets filled with an errno off the wire in response to the 32 * message they send. 33 * 34 * Handlers for unsolicited messages are registered. Each socket has a page 35 * that incoming data is copied into. First the header, then the data. 36 * Handlers are called from only one thread with a reference to this per-socket 37 * page. This page is destroyed after the handler call, so it can't be 38 * referenced beyond the call. Handlers may block but are discouraged from 39 * doing so. 40 * 41 * Any framing errors (bad magic, large payload lengths) close a connection. 42 * 43 * Our sock_container holds the state we associate with a socket. It's current 44 * framing state is held there as well as the refcounting we do around when it 45 * is safe to tear down the socket. The socket is only finally torn down from 46 * the container when the container loses all of its references -- so as long 47 * as you hold a ref on the container you can trust that the socket is valid 48 * for use with kernel socket APIs. 49 * 50 * Connections are initiated between a pair of nodes when the node with the 51 * higher node number gets a heartbeat callback which indicates that the lower 52 * numbered node has started heartbeating. The lower numbered node is passive 53 * and only accepts the connection if the higher numbered node is heartbeating. 54 */ 55 56#include <linux/kernel.h> 57#include <linux/jiffies.h> 58#include <linux/slab.h> 59#include <linux/idr.h> 60#include <linux/kref.h> 61#include <linux/net.h> 62#include <linux/export.h> 63#include <net/tcp.h> 64 65#include <asm/uaccess.h> 66 67#include "heartbeat.h" 68#include "tcp.h" 69#include "nodemanager.h" 70#define MLOG_MASK_PREFIX ML_TCP 71#include "masklog.h" 72#include "quorum.h" 73 74#include "tcp_internal.h" 75 76#define SC_NODEF_FMT "node %s (num %u) at %pI4:%u" 77#define SC_NODEF_ARGS(sc) sc->sc_node->nd_name, sc->sc_node->nd_num, \ 78 &sc->sc_node->nd_ipv4_address, \ 79 ntohs(sc->sc_node->nd_ipv4_port) 80 81/* 82 * In the following two log macros, the whitespace after the ',' just 83 * before ##args is intentional. Otherwise, gcc 2.95 will eat the 84 * previous token if args expands to nothing. 85 */ 86#define msglog(hdr, fmt, args...) do { \ 87 typeof(hdr) __hdr = (hdr); \ 88 mlog(ML_MSG, "[mag %u len %u typ %u stat %d sys_stat %d " \ 89 "key %08x num %u] " fmt, \ 90 be16_to_cpu(__hdr->magic), be16_to_cpu(__hdr->data_len), \ 91 be16_to_cpu(__hdr->msg_type), be32_to_cpu(__hdr->status), \ 92 be32_to_cpu(__hdr->sys_status), be32_to_cpu(__hdr->key), \ 93 be32_to_cpu(__hdr->msg_num) , ##args); \ 94} while (0) 95 96#define sclog(sc, fmt, args...) do { \ 97 typeof(sc) __sc = (sc); \ 98 mlog(ML_SOCKET, "[sc %p refs %d sock %p node %u page %p " \ 99 "pg_off %zu] " fmt, __sc, \ 100 atomic_read(&__sc->sc_kref.refcount), __sc->sc_sock, \ 101 __sc->sc_node->nd_num, __sc->sc_page, __sc->sc_page_off , \ 102 ##args); \ 103} while (0) 104 105static DEFINE_RWLOCK(o2net_handler_lock); 106static struct rb_root o2net_handler_tree = RB_ROOT; 107 108static struct o2net_node o2net_nodes[O2NM_MAX_NODES]; 109 110/* XXX someday we'll need better accounting */ 111static struct socket *o2net_listen_sock; 112 113/* 114 * listen work is only queued by the listening socket callbacks on the 115 * o2net_wq. teardown detaches the callbacks before destroying the workqueue. 116 * quorum work is queued as sock containers are shutdown.. stop_listening 117 * tears down all the node's sock containers, preventing future shutdowns 118 * and queued quroum work, before canceling delayed quorum work and 119 * destroying the work queue. 120 */ 121static struct workqueue_struct *o2net_wq; 122static struct work_struct o2net_listen_work; 123 124static struct o2hb_callback_func o2net_hb_up, o2net_hb_down; 125#define O2NET_HB_PRI 0x1 126 127static struct o2net_handshake *o2net_hand; 128static struct o2net_msg *o2net_keep_req, *o2net_keep_resp; 129 130static int o2net_sys_err_translations[O2NET_ERR_MAX] = 131 {[O2NET_ERR_NONE] = 0, 132 [O2NET_ERR_NO_HNDLR] = -ENOPROTOOPT, 133 [O2NET_ERR_OVERFLOW] = -EOVERFLOW, 134 [O2NET_ERR_DIED] = -EHOSTDOWN,}; 135 136/* can't quite avoid *all* internal declarations :/ */ 137static void o2net_sc_connect_completed(struct work_struct *work); 138static void o2net_rx_until_empty(struct work_struct *work); 139static void o2net_shutdown_sc(struct work_struct *work); 140static void o2net_listen_data_ready(struct sock *sk); 141static void o2net_sc_send_keep_req(struct work_struct *work); 142static void o2net_idle_timer(unsigned long data); 143static void o2net_sc_postpone_idle(struct o2net_sock_container *sc); 144static void o2net_sc_reset_idle_timer(struct o2net_sock_container *sc); 145 146#ifdef CONFIG_DEBUG_FS 147static void o2net_init_nst(struct o2net_send_tracking *nst, u32 msgtype, 148 u32 msgkey, struct task_struct *task, u8 node) 149{ 150 INIT_LIST_HEAD(&nst->st_net_debug_item); 151 nst->st_task = task; 152 nst->st_msg_type = msgtype; 153 nst->st_msg_key = msgkey; 154 nst->st_node = node; 155} 156 157static inline void o2net_set_nst_sock_time(struct o2net_send_tracking *nst) 158{ 159 nst->st_sock_time = ktime_get(); 160} 161 162static inline void o2net_set_nst_send_time(struct o2net_send_tracking *nst) 163{ 164 nst->st_send_time = ktime_get(); 165} 166 167static inline void o2net_set_nst_status_time(struct o2net_send_tracking *nst) 168{ 169 nst->st_status_time = ktime_get(); 170} 171 172static inline void o2net_set_nst_sock_container(struct o2net_send_tracking *nst, 173 struct o2net_sock_container *sc) 174{ 175 nst->st_sc = sc; 176} 177 178static inline void o2net_set_nst_msg_id(struct o2net_send_tracking *nst, 179 u32 msg_id) 180{ 181 nst->st_id = msg_id; 182} 183 184static inline void o2net_set_sock_timer(struct o2net_sock_container *sc) 185{ 186 sc->sc_tv_timer = ktime_get(); 187} 188 189static inline void o2net_set_data_ready_time(struct o2net_sock_container *sc) 190{ 191 sc->sc_tv_data_ready = ktime_get(); 192} 193 194static inline void o2net_set_advance_start_time(struct o2net_sock_container *sc) 195{ 196 sc->sc_tv_advance_start = ktime_get(); 197} 198 199static inline void o2net_set_advance_stop_time(struct o2net_sock_container *sc) 200{ 201 sc->sc_tv_advance_stop = ktime_get(); 202} 203 204static inline void o2net_set_func_start_time(struct o2net_sock_container *sc) 205{ 206 sc->sc_tv_func_start = ktime_get(); 207} 208 209static inline void o2net_set_func_stop_time(struct o2net_sock_container *sc) 210{ 211 sc->sc_tv_func_stop = ktime_get(); 212} 213 214#else /* CONFIG_DEBUG_FS */ 215# define o2net_init_nst(a, b, c, d, e) 216# define o2net_set_nst_sock_time(a) 217# define o2net_set_nst_send_time(a) 218# define o2net_set_nst_status_time(a) 219# define o2net_set_nst_sock_container(a, b) 220# define o2net_set_nst_msg_id(a, b) 221# define o2net_set_sock_timer(a) 222# define o2net_set_data_ready_time(a) 223# define o2net_set_advance_start_time(a) 224# define o2net_set_advance_stop_time(a) 225# define o2net_set_func_start_time(a) 226# define o2net_set_func_stop_time(a) 227#endif /* CONFIG_DEBUG_FS */ 228 229#ifdef CONFIG_OCFS2_FS_STATS 230static ktime_t o2net_get_func_run_time(struct o2net_sock_container *sc) 231{ 232 return ktime_sub(sc->sc_tv_func_stop, sc->sc_tv_func_start); 233} 234 235static void o2net_update_send_stats(struct o2net_send_tracking *nst, 236 struct o2net_sock_container *sc) 237{ 238 sc->sc_tv_status_total = ktime_add(sc->sc_tv_status_total, 239 ktime_sub(ktime_get(), 240 nst->st_status_time)); 241 sc->sc_tv_send_total = ktime_add(sc->sc_tv_send_total, 242 ktime_sub(nst->st_status_time, 243 nst->st_send_time)); 244 sc->sc_tv_acquiry_total = ktime_add(sc->sc_tv_acquiry_total, 245 ktime_sub(nst->st_send_time, 246 nst->st_sock_time)); 247 sc->sc_send_count++; 248} 249 250static void o2net_update_recv_stats(struct o2net_sock_container *sc) 251{ 252 sc->sc_tv_process_total = ktime_add(sc->sc_tv_process_total, 253 o2net_get_func_run_time(sc)); 254 sc->sc_recv_count++; 255} 256 257#else 258 259# define o2net_update_send_stats(a, b) 260 261# define o2net_update_recv_stats(sc) 262 263#endif /* CONFIG_OCFS2_FS_STATS */ 264 265static inline unsigned int o2net_reconnect_delay(void) 266{ 267 return o2nm_single_cluster->cl_reconnect_delay_ms; 268} 269 270static inline unsigned int o2net_keepalive_delay(void) 271{ 272 return o2nm_single_cluster->cl_keepalive_delay_ms; 273} 274 275static inline unsigned int o2net_idle_timeout(void) 276{ 277 return o2nm_single_cluster->cl_idle_timeout_ms; 278} 279 280static inline int o2net_sys_err_to_errno(enum o2net_system_error err) 281{ 282 int trans; 283 BUG_ON(err >= O2NET_ERR_MAX); 284 trans = o2net_sys_err_translations[err]; 285 286 /* Just in case we mess up the translation table above */ 287 BUG_ON(err != O2NET_ERR_NONE && trans == 0); 288 return trans; 289} 290 291static struct o2net_node * o2net_nn_from_num(u8 node_num) 292{ 293 BUG_ON(node_num >= ARRAY_SIZE(o2net_nodes)); 294 return &o2net_nodes[node_num]; 295} 296 297static u8 o2net_num_from_nn(struct o2net_node *nn) 298{ 299 BUG_ON(nn == NULL); 300 return nn - o2net_nodes; 301} 302 303/* ------------------------------------------------------------ */ 304 305static int o2net_prep_nsw(struct o2net_node *nn, struct o2net_status_wait *nsw) 306{ 307 int ret; 308 309 spin_lock(&nn->nn_lock); 310 ret = idr_alloc(&nn->nn_status_idr, nsw, 0, 0, GFP_ATOMIC); 311 if (ret >= 0) { 312 nsw->ns_id = ret; 313 list_add_tail(&nsw->ns_node_item, &nn->nn_status_list); 314 } 315 spin_unlock(&nn->nn_lock); 316 if (ret < 0) 317 return ret; 318 319 init_waitqueue_head(&nsw->ns_wq); 320 nsw->ns_sys_status = O2NET_ERR_NONE; 321 nsw->ns_status = 0; 322 return 0; 323} 324 325static void o2net_complete_nsw_locked(struct o2net_node *nn, 326 struct o2net_status_wait *nsw, 327 enum o2net_system_error sys_status, 328 s32 status) 329{ 330 assert_spin_locked(&nn->nn_lock); 331 332 if (!list_empty(&nsw->ns_node_item)) { 333 list_del_init(&nsw->ns_node_item); 334 nsw->ns_sys_status = sys_status; 335 nsw->ns_status = status; 336 idr_remove(&nn->nn_status_idr, nsw->ns_id); 337 wake_up(&nsw->ns_wq); 338 } 339} 340 341static void o2net_complete_nsw(struct o2net_node *nn, 342 struct o2net_status_wait *nsw, 343 u64 id, enum o2net_system_error sys_status, 344 s32 status) 345{ 346 spin_lock(&nn->nn_lock); 347 if (nsw == NULL) { 348 if (id > INT_MAX) 349 goto out; 350 351 nsw = idr_find(&nn->nn_status_idr, id); 352 if (nsw == NULL) 353 goto out; 354 } 355 356 o2net_complete_nsw_locked(nn, nsw, sys_status, status); 357 358out: 359 spin_unlock(&nn->nn_lock); 360 return; 361} 362 363static void o2net_complete_nodes_nsw(struct o2net_node *nn) 364{ 365 struct o2net_status_wait *nsw, *tmp; 366 unsigned int num_kills = 0; 367 368 assert_spin_locked(&nn->nn_lock); 369 370 list_for_each_entry_safe(nsw, tmp, &nn->nn_status_list, ns_node_item) { 371 o2net_complete_nsw_locked(nn, nsw, O2NET_ERR_DIED, 0); 372 num_kills++; 373 } 374 375 mlog(0, "completed %d messages for node %u\n", num_kills, 376 o2net_num_from_nn(nn)); 377} 378 379static int o2net_nsw_completed(struct o2net_node *nn, 380 struct o2net_status_wait *nsw) 381{ 382 int completed; 383 spin_lock(&nn->nn_lock); 384 completed = list_empty(&nsw->ns_node_item); 385 spin_unlock(&nn->nn_lock); 386 return completed; 387} 388 389/* ------------------------------------------------------------ */ 390 391static void sc_kref_release(struct kref *kref) 392{ 393 struct o2net_sock_container *sc = container_of(kref, 394 struct o2net_sock_container, sc_kref); 395 BUG_ON(timer_pending(&sc->sc_idle_timeout)); 396 397 sclog(sc, "releasing\n"); 398 399 if (sc->sc_sock) { 400 sock_release(sc->sc_sock); 401 sc->sc_sock = NULL; 402 } 403 404 o2nm_undepend_item(&sc->sc_node->nd_item); 405 o2nm_node_put(sc->sc_node); 406 sc->sc_node = NULL; 407 408 o2net_debug_del_sc(sc); 409 410 if (sc->sc_page) 411 __free_page(sc->sc_page); 412 kfree(sc); 413} 414 415static void sc_put(struct o2net_sock_container *sc) 416{ 417 sclog(sc, "put\n"); 418 kref_put(&sc->sc_kref, sc_kref_release); 419} 420static void sc_get(struct o2net_sock_container *sc) 421{ 422 sclog(sc, "get\n"); 423 kref_get(&sc->sc_kref); 424} 425static struct o2net_sock_container *sc_alloc(struct o2nm_node *node) 426{ 427 struct o2net_sock_container *sc, *ret = NULL; 428 struct page *page = NULL; 429 int status = 0; 430 431 page = alloc_page(GFP_NOFS); 432 sc = kzalloc(sizeof(*sc), GFP_NOFS); 433 if (sc == NULL || page == NULL) 434 goto out; 435 436 kref_init(&sc->sc_kref); 437 o2nm_node_get(node); 438 sc->sc_node = node; 439 440 /* pin the node item of the remote node */ 441 status = o2nm_depend_item(&node->nd_item); 442 if (status) { 443 mlog_errno(status); 444 o2nm_node_put(node); 445 goto out; 446 } 447 INIT_WORK(&sc->sc_connect_work, o2net_sc_connect_completed); 448 INIT_WORK(&sc->sc_rx_work, o2net_rx_until_empty); 449 INIT_WORK(&sc->sc_shutdown_work, o2net_shutdown_sc); 450 INIT_DELAYED_WORK(&sc->sc_keepalive_work, o2net_sc_send_keep_req); 451 452 init_timer(&sc->sc_idle_timeout); 453 sc->sc_idle_timeout.function = o2net_idle_timer; 454 sc->sc_idle_timeout.data = (unsigned long)sc; 455 456 sclog(sc, "alloced\n"); 457 458 ret = sc; 459 sc->sc_page = page; 460 o2net_debug_add_sc(sc); 461 sc = NULL; 462 page = NULL; 463 464out: 465 if (page) 466 __free_page(page); 467 kfree(sc); 468 469 return ret; 470} 471 472/* ------------------------------------------------------------ */ 473 474static void o2net_sc_queue_work(struct o2net_sock_container *sc, 475 struct work_struct *work) 476{ 477 sc_get(sc); 478 if (!queue_work(o2net_wq, work)) 479 sc_put(sc); 480} 481static void o2net_sc_queue_delayed_work(struct o2net_sock_container *sc, 482 struct delayed_work *work, 483 int delay) 484{ 485 sc_get(sc); 486 if (!queue_delayed_work(o2net_wq, work, delay)) 487 sc_put(sc); 488} 489static void o2net_sc_cancel_delayed_work(struct o2net_sock_container *sc, 490 struct delayed_work *work) 491{ 492 if (cancel_delayed_work(work)) 493 sc_put(sc); 494} 495 496static atomic_t o2net_connected_peers = ATOMIC_INIT(0); 497 498int o2net_num_connected_peers(void) 499{ 500 return atomic_read(&o2net_connected_peers); 501} 502 503static void o2net_set_nn_state(struct o2net_node *nn, 504 struct o2net_sock_container *sc, 505 unsigned valid, int err) 506{ 507 int was_valid = nn->nn_sc_valid; 508 int was_err = nn->nn_persistent_error; 509 struct o2net_sock_container *old_sc = nn->nn_sc; 510 511 assert_spin_locked(&nn->nn_lock); 512 513 if (old_sc && !sc) 514 atomic_dec(&o2net_connected_peers); 515 else if (!old_sc && sc) 516 atomic_inc(&o2net_connected_peers); 517 518 /* the node num comparison and single connect/accept path should stop 519 * an non-null sc from being overwritten with another */ 520 BUG_ON(sc && nn->nn_sc && nn->nn_sc != sc); 521 mlog_bug_on_msg(err && valid, "err %d valid %u\n", err, valid); 522 mlog_bug_on_msg(valid && !sc, "valid %u sc %p\n", valid, sc); 523 524 if (was_valid && !valid && err == 0) 525 err = -ENOTCONN; 526 527 mlog(ML_CONN, "node %u sc: %p -> %p, valid %u -> %u, err %d -> %d\n", 528 o2net_num_from_nn(nn), nn->nn_sc, sc, nn->nn_sc_valid, valid, 529 nn->nn_persistent_error, err); 530 531 nn->nn_sc = sc; 532 nn->nn_sc_valid = valid ? 1 : 0; 533 nn->nn_persistent_error = err; 534 535 /* mirrors o2net_tx_can_proceed() */ 536 if (nn->nn_persistent_error || nn->nn_sc_valid) 537 wake_up(&nn->nn_sc_wq); 538 539 if (was_valid && !was_err && nn->nn_persistent_error) { 540 o2quo_conn_err(o2net_num_from_nn(nn)); 541 queue_delayed_work(o2net_wq, &nn->nn_still_up, 542 msecs_to_jiffies(O2NET_QUORUM_DELAY_MS)); 543 } 544 545 if (was_valid && !valid) { 546 if (old_sc) 547 printk(KERN_NOTICE "o2net: No longer connected to " 548 SC_NODEF_FMT "\n", SC_NODEF_ARGS(old_sc)); 549 o2net_complete_nodes_nsw(nn); 550 } 551 552 if (!was_valid && valid) { 553 o2quo_conn_up(o2net_num_from_nn(nn)); 554 cancel_delayed_work(&nn->nn_connect_expired); 555 printk(KERN_NOTICE "o2net: %s " SC_NODEF_FMT "\n", 556 o2nm_this_node() > sc->sc_node->nd_num ? 557 "Connected to" : "Accepted connection from", 558 SC_NODEF_ARGS(sc)); 559 } 560 561 /* trigger the connecting worker func as long as we're not valid, 562 * it will back off if it shouldn't connect. This can be called 563 * from node config teardown and so needs to be careful about 564 * the work queue actually being up. */ 565 if (!valid && o2net_wq) { 566 unsigned long delay; 567 /* delay if we're within a RECONNECT_DELAY of the 568 * last attempt */ 569 delay = (nn->nn_last_connect_attempt + 570 msecs_to_jiffies(o2net_reconnect_delay())) 571 - jiffies; 572 if (delay > msecs_to_jiffies(o2net_reconnect_delay())) 573 delay = 0; 574 mlog(ML_CONN, "queueing conn attempt in %lu jiffies\n", delay); 575 queue_delayed_work(o2net_wq, &nn->nn_connect_work, delay); 576 577 /* 578 * Delay the expired work after idle timeout. 579 * 580 * We might have lots of failed connection attempts that run 581 * through here but we only cancel the connect_expired work when 582 * a connection attempt succeeds. So only the first enqueue of 583 * the connect_expired work will do anything. The rest will see 584 * that it's already queued and do nothing. 585 */ 586 delay += msecs_to_jiffies(o2net_idle_timeout()); 587 queue_delayed_work(o2net_wq, &nn->nn_connect_expired, delay); 588 } 589 590 /* keep track of the nn's sc ref for the caller */ 591 if ((old_sc == NULL) && sc) 592 sc_get(sc); 593 if (old_sc && (old_sc != sc)) { 594 o2net_sc_queue_work(old_sc, &old_sc->sc_shutdown_work); 595 sc_put(old_sc); 596 } 597} 598 599/* see o2net_register_callbacks() */ 600static void o2net_data_ready(struct sock *sk) 601{ 602 void (*ready)(struct sock *sk); 603 604 read_lock(&sk->sk_callback_lock); 605 if (sk->sk_user_data) { 606 struct o2net_sock_container *sc = sk->sk_user_data; 607 sclog(sc, "data_ready hit\n"); 608 o2net_set_data_ready_time(sc); 609 o2net_sc_queue_work(sc, &sc->sc_rx_work); 610 ready = sc->sc_data_ready; 611 } else { 612 ready = sk->sk_data_ready; 613 } 614 read_unlock(&sk->sk_callback_lock); 615 616 ready(sk); 617} 618 619/* see o2net_register_callbacks() */ 620static void o2net_state_change(struct sock *sk) 621{ 622 void (*state_change)(struct sock *sk); 623 struct o2net_sock_container *sc; 624 625 read_lock(&sk->sk_callback_lock); 626 sc = sk->sk_user_data; 627 if (sc == NULL) { 628 state_change = sk->sk_state_change; 629 goto out; 630 } 631 632 sclog(sc, "state_change to %d\n", sk->sk_state); 633 634 state_change = sc->sc_state_change; 635 636 switch(sk->sk_state) { 637 /* ignore connecting sockets as they make progress */ 638 case TCP_SYN_SENT: 639 case TCP_SYN_RECV: 640 break; 641 case TCP_ESTABLISHED: 642 o2net_sc_queue_work(sc, &sc->sc_connect_work); 643 break; 644 default: 645 printk(KERN_INFO "o2net: Connection to " SC_NODEF_FMT 646 " shutdown, state %d\n", 647 SC_NODEF_ARGS(sc), sk->sk_state); 648 o2net_sc_queue_work(sc, &sc->sc_shutdown_work); 649 break; 650 } 651out: 652 read_unlock(&sk->sk_callback_lock); 653 state_change(sk); 654} 655 656/* 657 * we register callbacks so we can queue work on events before calling 658 * the original callbacks. our callbacks our careful to test user_data 659 * to discover when they've reaced with o2net_unregister_callbacks(). 660 */ 661static void o2net_register_callbacks(struct sock *sk, 662 struct o2net_sock_container *sc) 663{ 664 write_lock_bh(&sk->sk_callback_lock); 665 666 /* accepted sockets inherit the old listen socket data ready */ 667 if (sk->sk_data_ready == o2net_listen_data_ready) { 668 sk->sk_data_ready = sk->sk_user_data; 669 sk->sk_user_data = NULL; 670 } 671 672 BUG_ON(sk->sk_user_data != NULL); 673 sk->sk_user_data = sc; 674 sc_get(sc); 675 676 sc->sc_data_ready = sk->sk_data_ready; 677 sc->sc_state_change = sk->sk_state_change; 678 sk->sk_data_ready = o2net_data_ready; 679 sk->sk_state_change = o2net_state_change; 680 681 mutex_init(&sc->sc_send_lock); 682 683 write_unlock_bh(&sk->sk_callback_lock); 684} 685 686static int o2net_unregister_callbacks(struct sock *sk, 687 struct o2net_sock_container *sc) 688{ 689 int ret = 0; 690 691 write_lock_bh(&sk->sk_callback_lock); 692 if (sk->sk_user_data == sc) { 693 ret = 1; 694 sk->sk_user_data = NULL; 695 sk->sk_data_ready = sc->sc_data_ready; 696 sk->sk_state_change = sc->sc_state_change; 697 } 698 write_unlock_bh(&sk->sk_callback_lock); 699 700 return ret; 701} 702 703/* 704 * this is a little helper that is called by callers who have seen a problem 705 * with an sc and want to detach it from the nn if someone already hasn't beat 706 * them to it. if an error is given then the shutdown will be persistent 707 * and pending transmits will be canceled. 708 */ 709static void o2net_ensure_shutdown(struct o2net_node *nn, 710 struct o2net_sock_container *sc, 711 int err) 712{ 713 spin_lock(&nn->nn_lock); 714 if (nn->nn_sc == sc) 715 o2net_set_nn_state(nn, NULL, 0, err); 716 spin_unlock(&nn->nn_lock); 717} 718 719/* 720 * This work queue function performs the blocking parts of socket shutdown. A 721 * few paths lead here. set_nn_state will trigger this callback if it sees an 722 * sc detached from the nn. state_change will also trigger this callback 723 * directly when it sees errors. In that case we need to call set_nn_state 724 * ourselves as state_change couldn't get the nn_lock and call set_nn_state 725 * itself. 726 */ 727static void o2net_shutdown_sc(struct work_struct *work) 728{ 729 struct o2net_sock_container *sc = 730 container_of(work, struct o2net_sock_container, 731 sc_shutdown_work); 732 struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); 733 734 sclog(sc, "shutting down\n"); 735 736 /* drop the callbacks ref and call shutdown only once */ 737 if (o2net_unregister_callbacks(sc->sc_sock->sk, sc)) { 738 /* we shouldn't flush as we're in the thread, the 739 * races with pending sc work structs are harmless */ 740 del_timer_sync(&sc->sc_idle_timeout); 741 o2net_sc_cancel_delayed_work(sc, &sc->sc_keepalive_work); 742 sc_put(sc); 743 kernel_sock_shutdown(sc->sc_sock, SHUT_RDWR); 744 } 745 746 /* not fatal so failed connects before the other guy has our 747 * heartbeat can be retried */ 748 o2net_ensure_shutdown(nn, sc, 0); 749 sc_put(sc); 750} 751 752/* ------------------------------------------------------------ */ 753 754static int o2net_handler_cmp(struct o2net_msg_handler *nmh, u32 msg_type, 755 u32 key) 756{ 757 int ret = memcmp(&nmh->nh_key, &key, sizeof(key)); 758 759 if (ret == 0) 760 ret = memcmp(&nmh->nh_msg_type, &msg_type, sizeof(msg_type)); 761 762 return ret; 763} 764 765static struct o2net_msg_handler * 766o2net_handler_tree_lookup(u32 msg_type, u32 key, struct rb_node ***ret_p, 767 struct rb_node **ret_parent) 768{ 769 struct rb_node **p = &o2net_handler_tree.rb_node; 770 struct rb_node *parent = NULL; 771 struct o2net_msg_handler *nmh, *ret = NULL; 772 int cmp; 773 774 while (*p) { 775 parent = *p; 776 nmh = rb_entry(parent, struct o2net_msg_handler, nh_node); 777 cmp = o2net_handler_cmp(nmh, msg_type, key); 778 779 if (cmp < 0) 780 p = &(*p)->rb_left; 781 else if (cmp > 0) 782 p = &(*p)->rb_right; 783 else { 784 ret = nmh; 785 break; 786 } 787 } 788 789 if (ret_p != NULL) 790 *ret_p = p; 791 if (ret_parent != NULL) 792 *ret_parent = parent; 793 794 return ret; 795} 796 797static void o2net_handler_kref_release(struct kref *kref) 798{ 799 struct o2net_msg_handler *nmh; 800 nmh = container_of(kref, struct o2net_msg_handler, nh_kref); 801 802 kfree(nmh); 803} 804 805static void o2net_handler_put(struct o2net_msg_handler *nmh) 806{ 807 kref_put(&nmh->nh_kref, o2net_handler_kref_release); 808} 809 810/* max_len is protection for the handler func. incoming messages won't 811 * be given to the handler if their payload is longer than the max. */ 812int o2net_register_handler(u32 msg_type, u32 key, u32 max_len, 813 o2net_msg_handler_func *func, void *data, 814 o2net_post_msg_handler_func *post_func, 815 struct list_head *unreg_list) 816{ 817 struct o2net_msg_handler *nmh = NULL; 818 struct rb_node **p, *parent; 819 int ret = 0; 820 821 if (max_len > O2NET_MAX_PAYLOAD_BYTES) { 822 mlog(0, "max_len for message handler out of range: %u\n", 823 max_len); 824 ret = -EINVAL; 825 goto out; 826 } 827 828 if (!msg_type) { 829 mlog(0, "no message type provided: %u, %p\n", msg_type, func); 830 ret = -EINVAL; 831 goto out; 832 833 } 834 if (!func) { 835 mlog(0, "no message handler provided: %u, %p\n", 836 msg_type, func); 837 ret = -EINVAL; 838 goto out; 839 } 840 841 nmh = kzalloc(sizeof(struct o2net_msg_handler), GFP_NOFS); 842 if (nmh == NULL) { 843 ret = -ENOMEM; 844 goto out; 845 } 846 847 nmh->nh_func = func; 848 nmh->nh_func_data = data; 849 nmh->nh_post_func = post_func; 850 nmh->nh_msg_type = msg_type; 851 nmh->nh_max_len = max_len; 852 nmh->nh_key = key; 853 /* the tree and list get this ref.. they're both removed in 854 * unregister when this ref is dropped */ 855 kref_init(&nmh->nh_kref); 856 INIT_LIST_HEAD(&nmh->nh_unregister_item); 857 858 write_lock(&o2net_handler_lock); 859 if (o2net_handler_tree_lookup(msg_type, key, &p, &parent)) 860 ret = -EEXIST; 861 else { 862 rb_link_node(&nmh->nh_node, parent, p); 863 rb_insert_color(&nmh->nh_node, &o2net_handler_tree); 864 list_add_tail(&nmh->nh_unregister_item, unreg_list); 865 866 mlog(ML_TCP, "registered handler func %p type %u key %08x\n", 867 func, msg_type, key); 868 /* we've had some trouble with handlers seemingly vanishing. */ 869 mlog_bug_on_msg(o2net_handler_tree_lookup(msg_type, key, &p, 870 &parent) == NULL, 871 "couldn't find handler we *just* registered " 872 "for type %u key %08x\n", msg_type, key); 873 } 874 write_unlock(&o2net_handler_lock); 875 if (ret) 876 goto out; 877 878out: 879 if (ret) 880 kfree(nmh); 881 882 return ret; 883} 884EXPORT_SYMBOL_GPL(o2net_register_handler); 885 886void o2net_unregister_handler_list(struct list_head *list) 887{ 888 struct o2net_msg_handler *nmh, *n; 889 890 write_lock(&o2net_handler_lock); 891 list_for_each_entry_safe(nmh, n, list, nh_unregister_item) { 892 mlog(ML_TCP, "unregistering handler func %p type %u key %08x\n", 893 nmh->nh_func, nmh->nh_msg_type, nmh->nh_key); 894 rb_erase(&nmh->nh_node, &o2net_handler_tree); 895 list_del_init(&nmh->nh_unregister_item); 896 kref_put(&nmh->nh_kref, o2net_handler_kref_release); 897 } 898 write_unlock(&o2net_handler_lock); 899} 900EXPORT_SYMBOL_GPL(o2net_unregister_handler_list); 901 902static struct o2net_msg_handler *o2net_handler_get(u32 msg_type, u32 key) 903{ 904 struct o2net_msg_handler *nmh; 905 906 read_lock(&o2net_handler_lock); 907 nmh = o2net_handler_tree_lookup(msg_type, key, NULL, NULL); 908 if (nmh) 909 kref_get(&nmh->nh_kref); 910 read_unlock(&o2net_handler_lock); 911 912 return nmh; 913} 914 915/* ------------------------------------------------------------ */ 916 917static int o2net_recv_tcp_msg(struct socket *sock, void *data, size_t len) 918{ 919 struct kvec vec = { .iov_len = len, .iov_base = data, }; 920 struct msghdr msg = { .msg_flags = MSG_DONTWAIT, }; 921 return kernel_recvmsg(sock, &msg, &vec, 1, len, msg.msg_flags); 922} 923 924static int o2net_send_tcp_msg(struct socket *sock, struct kvec *vec, 925 size_t veclen, size_t total) 926{ 927 int ret; 928 struct msghdr msg = {.msg_flags = 0,}; 929 930 if (sock == NULL) { 931 ret = -EINVAL; 932 goto out; 933 } 934 935 ret = kernel_sendmsg(sock, &msg, vec, veclen, total); 936 if (likely(ret == total)) 937 return 0; 938 mlog(ML_ERROR, "sendmsg returned %d instead of %zu\n", ret, total); 939 if (ret >= 0) 940 ret = -EPIPE; /* should be smarter, I bet */ 941out: 942 mlog(0, "returning error: %d\n", ret); 943 return ret; 944} 945 946static void o2net_sendpage(struct o2net_sock_container *sc, 947 void *kmalloced_virt, 948 size_t size) 949{ 950 struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); 951 ssize_t ret; 952 953 while (1) { 954 mutex_lock(&sc->sc_send_lock); 955 ret = sc->sc_sock->ops->sendpage(sc->sc_sock, 956 virt_to_page(kmalloced_virt), 957 (long)kmalloced_virt & ~PAGE_MASK, 958 size, MSG_DONTWAIT); 959 mutex_unlock(&sc->sc_send_lock); 960 if (ret == size) 961 break; 962 if (ret == (ssize_t)-EAGAIN) { 963 mlog(0, "sendpage of size %zu to " SC_NODEF_FMT 964 " returned EAGAIN\n", size, SC_NODEF_ARGS(sc)); 965 cond_resched(); 966 continue; 967 } 968 mlog(ML_ERROR, "sendpage of size %zu to " SC_NODEF_FMT 969 " failed with %zd\n", size, SC_NODEF_ARGS(sc), ret); 970 o2net_ensure_shutdown(nn, sc, 0); 971 break; 972 } 973} 974 975static void o2net_init_msg(struct o2net_msg *msg, u16 data_len, u16 msg_type, u32 key) 976{ 977 memset(msg, 0, sizeof(struct o2net_msg)); 978 msg->magic = cpu_to_be16(O2NET_MSG_MAGIC); 979 msg->data_len = cpu_to_be16(data_len); 980 msg->msg_type = cpu_to_be16(msg_type); 981 msg->sys_status = cpu_to_be32(O2NET_ERR_NONE); 982 msg->status = 0; 983 msg->key = cpu_to_be32(key); 984} 985 986static int o2net_tx_can_proceed(struct o2net_node *nn, 987 struct o2net_sock_container **sc_ret, 988 int *error) 989{ 990 int ret = 0; 991 992 spin_lock(&nn->nn_lock); 993 if (nn->nn_persistent_error) { 994 ret = 1; 995 *sc_ret = NULL; 996 *error = nn->nn_persistent_error; 997 } else if (nn->nn_sc_valid) { 998 kref_get(&nn->nn_sc->sc_kref); 999 1000 ret = 1; 1001 *sc_ret = nn->nn_sc; 1002 *error = 0; 1003 } 1004 spin_unlock(&nn->nn_lock); 1005 1006 return ret; 1007} 1008 1009/* Get a map of all nodes to which this node is currently connected to */ 1010void o2net_fill_node_map(unsigned long *map, unsigned bytes) 1011{ 1012 struct o2net_sock_container *sc; 1013 int node, ret; 1014 1015 BUG_ON(bytes < (BITS_TO_LONGS(O2NM_MAX_NODES) * sizeof(unsigned long))); 1016 1017 memset(map, 0, bytes); 1018 for (node = 0; node < O2NM_MAX_NODES; ++node) { 1019 if (!o2net_tx_can_proceed(o2net_nn_from_num(node), &sc, &ret)) 1020 continue; 1021 if (!ret) { 1022 set_bit(node, map); 1023 sc_put(sc); 1024 } 1025 } 1026} 1027EXPORT_SYMBOL_GPL(o2net_fill_node_map); 1028 1029int o2net_send_message_vec(u32 msg_type, u32 key, struct kvec *caller_vec, 1030 size_t caller_veclen, u8 target_node, int *status) 1031{ 1032 int ret = 0; 1033 struct o2net_msg *msg = NULL; 1034 size_t veclen, caller_bytes = 0; 1035 struct kvec *vec = NULL; 1036 struct o2net_sock_container *sc = NULL; 1037 struct o2net_node *nn = o2net_nn_from_num(target_node); 1038 struct o2net_status_wait nsw = { 1039 .ns_node_item = LIST_HEAD_INIT(nsw.ns_node_item), 1040 }; 1041 struct o2net_send_tracking nst; 1042 1043 o2net_init_nst(&nst, msg_type, key, current, target_node); 1044 1045 if (o2net_wq == NULL) { 1046 mlog(0, "attempt to tx without o2netd running\n"); 1047 ret = -ESRCH; 1048 goto out; 1049 } 1050 1051 if (caller_veclen == 0) { 1052 mlog(0, "bad kvec array length\n"); 1053 ret = -EINVAL; 1054 goto out; 1055 } 1056 1057 caller_bytes = iov_length((struct iovec *)caller_vec, caller_veclen); 1058 if (caller_bytes > O2NET_MAX_PAYLOAD_BYTES) { 1059 mlog(0, "total payload len %zu too large\n", caller_bytes); 1060 ret = -EINVAL; 1061 goto out; 1062 } 1063 1064 if (target_node == o2nm_this_node()) { 1065 ret = -ELOOP; 1066 goto out; 1067 } 1068 1069 o2net_debug_add_nst(&nst); 1070 1071 o2net_set_nst_sock_time(&nst); 1072 1073 wait_event(nn->nn_sc_wq, o2net_tx_can_proceed(nn, &sc, &ret)); 1074 if (ret) 1075 goto out; 1076 1077 o2net_set_nst_sock_container(&nst, sc); 1078 1079 veclen = caller_veclen + 1; 1080 vec = kmalloc(sizeof(struct kvec) * veclen, GFP_ATOMIC); 1081 if (vec == NULL) { 1082 mlog(0, "failed to %zu element kvec!\n", veclen); 1083 ret = -ENOMEM; 1084 goto out; 1085 } 1086 1087 msg = kmalloc(sizeof(struct o2net_msg), GFP_ATOMIC); 1088 if (!msg) { 1089 mlog(0, "failed to allocate a o2net_msg!\n"); 1090 ret = -ENOMEM; 1091 goto out; 1092 } 1093 1094 o2net_init_msg(msg, caller_bytes, msg_type, key); 1095 1096 vec[0].iov_len = sizeof(struct o2net_msg); 1097 vec[0].iov_base = msg; 1098 memcpy(&vec[1], caller_vec, caller_veclen * sizeof(struct kvec)); 1099 1100 ret = o2net_prep_nsw(nn, &nsw); 1101 if (ret) 1102 goto out; 1103 1104 msg->msg_num = cpu_to_be32(nsw.ns_id); 1105 o2net_set_nst_msg_id(&nst, nsw.ns_id); 1106 1107 o2net_set_nst_send_time(&nst); 1108 1109 /* finally, convert the message header to network byte-order 1110 * and send */ 1111 mutex_lock(&sc->sc_send_lock); 1112 ret = o2net_send_tcp_msg(sc->sc_sock, vec, veclen, 1113 sizeof(struct o2net_msg) + caller_bytes); 1114 mutex_unlock(&sc->sc_send_lock); 1115 msglog(msg, "sending returned %d\n", ret); 1116 if (ret < 0) { 1117 mlog(0, "error returned from o2net_send_tcp_msg=%d\n", ret); 1118 goto out; 1119 } 1120 1121 /* wait on other node's handler */ 1122 o2net_set_nst_status_time(&nst); 1123 wait_event(nsw.ns_wq, o2net_nsw_completed(nn, &nsw)); 1124 1125 o2net_update_send_stats(&nst, sc); 1126 1127 /* Note that we avoid overwriting the callers status return 1128 * variable if a system error was reported on the other 1129 * side. Callers beware. */ 1130 ret = o2net_sys_err_to_errno(nsw.ns_sys_status); 1131 if (status && !ret) 1132 *status = nsw.ns_status; 1133 1134 mlog(0, "woken, returning system status %d, user status %d\n", 1135 ret, nsw.ns_status); 1136out: 1137 o2net_debug_del_nst(&nst); /* must be before dropping sc and node */ 1138 if (sc) 1139 sc_put(sc); 1140 kfree(vec); 1141 kfree(msg); 1142 o2net_complete_nsw(nn, &nsw, 0, 0, 0); 1143 return ret; 1144} 1145EXPORT_SYMBOL_GPL(o2net_send_message_vec); 1146 1147int o2net_send_message(u32 msg_type, u32 key, void *data, u32 len, 1148 u8 target_node, int *status) 1149{ 1150 struct kvec vec = { 1151 .iov_base = data, 1152 .iov_len = len, 1153 }; 1154 return o2net_send_message_vec(msg_type, key, &vec, 1, 1155 target_node, status); 1156} 1157EXPORT_SYMBOL_GPL(o2net_send_message); 1158 1159static int o2net_send_status_magic(struct socket *sock, struct o2net_msg *hdr, 1160 enum o2net_system_error syserr, int err) 1161{ 1162 struct kvec vec = { 1163 .iov_base = hdr, 1164 .iov_len = sizeof(struct o2net_msg), 1165 }; 1166 1167 BUG_ON(syserr >= O2NET_ERR_MAX); 1168 1169 /* leave other fields intact from the incoming message, msg_num 1170 * in particular */ 1171 hdr->sys_status = cpu_to_be32(syserr); 1172 hdr->status = cpu_to_be32(err); 1173 hdr->magic = cpu_to_be16(O2NET_MSG_STATUS_MAGIC); // twiddle the magic 1174 hdr->data_len = 0; 1175 1176 msglog(hdr, "about to send status magic %d\n", err); 1177 /* hdr has been in host byteorder this whole time */ 1178 return o2net_send_tcp_msg(sock, &vec, 1, sizeof(struct o2net_msg)); 1179} 1180 1181/* this returns -errno if the header was unknown or too large, etc. 1182 * after this is called the buffer us reused for the next message */ 1183static int o2net_process_message(struct o2net_sock_container *sc, 1184 struct o2net_msg *hdr) 1185{ 1186 struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); 1187 int ret = 0, handler_status; 1188 enum o2net_system_error syserr; 1189 struct o2net_msg_handler *nmh = NULL; 1190 void *ret_data = NULL; 1191 1192 msglog(hdr, "processing message\n"); 1193 1194 o2net_sc_postpone_idle(sc); 1195 1196 switch(be16_to_cpu(hdr->magic)) { 1197 case O2NET_MSG_STATUS_MAGIC: 1198 /* special type for returning message status */ 1199 o2net_complete_nsw(nn, NULL, 1200 be32_to_cpu(hdr->msg_num), 1201 be32_to_cpu(hdr->sys_status), 1202 be32_to_cpu(hdr->status)); 1203 goto out; 1204 case O2NET_MSG_KEEP_REQ_MAGIC: 1205 o2net_sendpage(sc, o2net_keep_resp, 1206 sizeof(*o2net_keep_resp)); 1207 goto out; 1208 case O2NET_MSG_KEEP_RESP_MAGIC: 1209 goto out; 1210 case O2NET_MSG_MAGIC: 1211 break; 1212 default: 1213 msglog(hdr, "bad magic\n"); 1214 ret = -EINVAL; 1215 goto out; 1216 break; 1217 } 1218 1219 /* find a handler for it */ 1220 handler_status = 0; 1221 nmh = o2net_handler_get(be16_to_cpu(hdr->msg_type), 1222 be32_to_cpu(hdr->key)); 1223 if (!nmh) { 1224 mlog(ML_TCP, "couldn't find handler for type %u key %08x\n", 1225 be16_to_cpu(hdr->msg_type), be32_to_cpu(hdr->key)); 1226 syserr = O2NET_ERR_NO_HNDLR; 1227 goto out_respond; 1228 } 1229 1230 syserr = O2NET_ERR_NONE; 1231 1232 if (be16_to_cpu(hdr->data_len) > nmh->nh_max_len) 1233 syserr = O2NET_ERR_OVERFLOW; 1234 1235 if (syserr != O2NET_ERR_NONE) 1236 goto out_respond; 1237 1238 o2net_set_func_start_time(sc); 1239 sc->sc_msg_key = be32_to_cpu(hdr->key); 1240 sc->sc_msg_type = be16_to_cpu(hdr->msg_type); 1241 handler_status = (nmh->nh_func)(hdr, sizeof(struct o2net_msg) + 1242 be16_to_cpu(hdr->data_len), 1243 nmh->nh_func_data, &ret_data); 1244 o2net_set_func_stop_time(sc); 1245 1246 o2net_update_recv_stats(sc); 1247 1248out_respond: 1249 /* this destroys the hdr, so don't use it after this */ 1250 mutex_lock(&sc->sc_send_lock); 1251 ret = o2net_send_status_magic(sc->sc_sock, hdr, syserr, 1252 handler_status); 1253 mutex_unlock(&sc->sc_send_lock); 1254 hdr = NULL; 1255 mlog(0, "sending handler status %d, syserr %d returned %d\n", 1256 handler_status, syserr, ret); 1257 1258 if (nmh) { 1259 BUG_ON(ret_data != NULL && nmh->nh_post_func == NULL); 1260 if (nmh->nh_post_func) 1261 (nmh->nh_post_func)(handler_status, nmh->nh_func_data, 1262 ret_data); 1263 } 1264 1265out: 1266 if (nmh) 1267 o2net_handler_put(nmh); 1268 return ret; 1269} 1270 1271static int o2net_check_handshake(struct o2net_sock_container *sc) 1272{ 1273 struct o2net_handshake *hand = page_address(sc->sc_page); 1274 struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); 1275 1276 if (hand->protocol_version != cpu_to_be64(O2NET_PROTOCOL_VERSION)) { 1277 printk(KERN_NOTICE "o2net: " SC_NODEF_FMT " Advertised net " 1278 "protocol version %llu but %llu is required. " 1279 "Disconnecting.\n", SC_NODEF_ARGS(sc), 1280 (unsigned long long)be64_to_cpu(hand->protocol_version), 1281 O2NET_PROTOCOL_VERSION); 1282 1283 /* don't bother reconnecting if its the wrong version. */ 1284 o2net_ensure_shutdown(nn, sc, -ENOTCONN); 1285 return -1; 1286 } 1287 1288 /* 1289 * Ensure timeouts are consistent with other nodes, otherwise 1290 * we can end up with one node thinking that the other must be down, 1291 * but isn't. This can ultimately cause corruption. 1292 */ 1293 if (be32_to_cpu(hand->o2net_idle_timeout_ms) != 1294 o2net_idle_timeout()) { 1295 printk(KERN_NOTICE "o2net: " SC_NODEF_FMT " uses a network " 1296 "idle timeout of %u ms, but we use %u ms locally. " 1297 "Disconnecting.\n", SC_NODEF_ARGS(sc), 1298 be32_to_cpu(hand->o2net_idle_timeout_ms), 1299 o2net_idle_timeout()); 1300 o2net_ensure_shutdown(nn, sc, -ENOTCONN); 1301 return -1; 1302 } 1303 1304 if (be32_to_cpu(hand->o2net_keepalive_delay_ms) != 1305 o2net_keepalive_delay()) { 1306 printk(KERN_NOTICE "o2net: " SC_NODEF_FMT " uses a keepalive " 1307 "delay of %u ms, but we use %u ms locally. " 1308 "Disconnecting.\n", SC_NODEF_ARGS(sc), 1309 be32_to_cpu(hand->o2net_keepalive_delay_ms), 1310 o2net_keepalive_delay()); 1311 o2net_ensure_shutdown(nn, sc, -ENOTCONN); 1312 return -1; 1313 } 1314 1315 if (be32_to_cpu(hand->o2hb_heartbeat_timeout_ms) != 1316 O2HB_MAX_WRITE_TIMEOUT_MS) { 1317 printk(KERN_NOTICE "o2net: " SC_NODEF_FMT " uses a heartbeat " 1318 "timeout of %u ms, but we use %u ms locally. " 1319 "Disconnecting.\n", SC_NODEF_ARGS(sc), 1320 be32_to_cpu(hand->o2hb_heartbeat_timeout_ms), 1321 O2HB_MAX_WRITE_TIMEOUT_MS); 1322 o2net_ensure_shutdown(nn, sc, -ENOTCONN); 1323 return -1; 1324 } 1325 1326 sc->sc_handshake_ok = 1; 1327 1328 spin_lock(&nn->nn_lock); 1329 /* set valid and queue the idle timers only if it hasn't been 1330 * shut down already */ 1331 if (nn->nn_sc == sc) { 1332 o2net_sc_reset_idle_timer(sc); 1333 atomic_set(&nn->nn_timeout, 0); 1334 o2net_set_nn_state(nn, sc, 1, 0); 1335 } 1336 spin_unlock(&nn->nn_lock); 1337 1338 /* shift everything up as though it wasn't there */ 1339 sc->sc_page_off -= sizeof(struct o2net_handshake); 1340 if (sc->sc_page_off) 1341 memmove(hand, hand + 1, sc->sc_page_off); 1342 1343 return 0; 1344} 1345 1346/* this demuxes the queued rx bytes into header or payload bits and calls 1347 * handlers as each full message is read off the socket. it returns -error, 1348 * == 0 eof, or > 0 for progress made.*/ 1349static int o2net_advance_rx(struct o2net_sock_container *sc) 1350{ 1351 struct o2net_msg *hdr; 1352 int ret = 0; 1353 void *data; 1354 size_t datalen; 1355 1356 sclog(sc, "receiving\n"); 1357 o2net_set_advance_start_time(sc); 1358 1359 if (unlikely(sc->sc_handshake_ok == 0)) { 1360 if(sc->sc_page_off < sizeof(struct o2net_handshake)) { 1361 data = page_address(sc->sc_page) + sc->sc_page_off; 1362 datalen = sizeof(struct o2net_handshake) - sc->sc_page_off; 1363 ret = o2net_recv_tcp_msg(sc->sc_sock, data, datalen); 1364 if (ret > 0) 1365 sc->sc_page_off += ret; 1366 } 1367 1368 if (sc->sc_page_off == sizeof(struct o2net_handshake)) { 1369 o2net_check_handshake(sc); 1370 if (unlikely(sc->sc_handshake_ok == 0)) 1371 ret = -EPROTO; 1372 } 1373 goto out; 1374 } 1375 1376 /* do we need more header? */ 1377 if (sc->sc_page_off < sizeof(struct o2net_msg)) { 1378 data = page_address(sc->sc_page) + sc->sc_page_off; 1379 datalen = sizeof(struct o2net_msg) - sc->sc_page_off; 1380 ret = o2net_recv_tcp_msg(sc->sc_sock, data, datalen); 1381 if (ret > 0) { 1382 sc->sc_page_off += ret; 1383 /* only swab incoming here.. we can 1384 * only get here once as we cross from 1385 * being under to over */ 1386 if (sc->sc_page_off == sizeof(struct o2net_msg)) { 1387 hdr = page_address(sc->sc_page); 1388 if (be16_to_cpu(hdr->data_len) > 1389 O2NET_MAX_PAYLOAD_BYTES) 1390 ret = -EOVERFLOW; 1391 } 1392 } 1393 if (ret <= 0) 1394 goto out; 1395 } 1396 1397 if (sc->sc_page_off < sizeof(struct o2net_msg)) { 1398 /* oof, still don't have a header */ 1399 goto out; 1400 } 1401 1402 /* this was swabbed above when we first read it */ 1403 hdr = page_address(sc->sc_page); 1404 1405 msglog(hdr, "at page_off %zu\n", sc->sc_page_off); 1406 1407 /* do we need more payload? */ 1408 if (sc->sc_page_off - sizeof(struct o2net_msg) < be16_to_cpu(hdr->data_len)) { 1409 /* need more payload */ 1410 data = page_address(sc->sc_page) + sc->sc_page_off; 1411 datalen = (sizeof(struct o2net_msg) + be16_to_cpu(hdr->data_len)) - 1412 sc->sc_page_off; 1413 ret = o2net_recv_tcp_msg(sc->sc_sock, data, datalen); 1414 if (ret > 0) 1415 sc->sc_page_off += ret; 1416 if (ret <= 0) 1417 goto out; 1418 } 1419 1420 if (sc->sc_page_off - sizeof(struct o2net_msg) == be16_to_cpu(hdr->data_len)) { 1421 /* we can only get here once, the first time we read 1422 * the payload.. so set ret to progress if the handler 1423 * works out. after calling this the message is toast */ 1424 ret = o2net_process_message(sc, hdr); 1425 if (ret == 0) 1426 ret = 1; 1427 sc->sc_page_off = 0; 1428 } 1429 1430out: 1431 sclog(sc, "ret = %d\n", ret); 1432 o2net_set_advance_stop_time(sc); 1433 return ret; 1434} 1435 1436/* this work func is triggerd by data ready. it reads until it can read no 1437 * more. it interprets 0, eof, as fatal. if data_ready hits while we're doing 1438 * our work the work struct will be marked and we'll be called again. */ 1439static void o2net_rx_until_empty(struct work_struct *work) 1440{ 1441 struct o2net_sock_container *sc = 1442 container_of(work, struct o2net_sock_container, sc_rx_work); 1443 int ret; 1444 1445 do { 1446 ret = o2net_advance_rx(sc); 1447 } while (ret > 0); 1448 1449 if (ret <= 0 && ret != -EAGAIN) { 1450 struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); 1451 sclog(sc, "saw error %d, closing\n", ret); 1452 /* not permanent so read failed handshake can retry */ 1453 o2net_ensure_shutdown(nn, sc, 0); 1454 } 1455 1456 sc_put(sc); 1457} 1458 1459static int o2net_set_nodelay(struct socket *sock) 1460{ 1461 int ret, val = 1; 1462 mm_segment_t oldfs; 1463 1464 oldfs = get_fs(); 1465 set_fs(KERNEL_DS); 1466 1467 /* 1468 * Dear unsuspecting programmer, 1469 * 1470 * Don't use sock_setsockopt() for SOL_TCP. It doesn't check its level 1471 * argument and assumes SOL_SOCKET so, say, your TCP_NODELAY will 1472 * silently turn into SO_DEBUG. 1473 * 1474 * Yours, 1475 * Keeper of hilariously fragile interfaces. 1476 */ 1477 ret = sock->ops->setsockopt(sock, SOL_TCP, TCP_NODELAY, 1478 (char __user *)&val, sizeof(val)); 1479 1480 set_fs(oldfs); 1481 return ret; 1482} 1483 1484static int o2net_set_usertimeout(struct socket *sock) 1485{ 1486 int user_timeout = O2NET_TCP_USER_TIMEOUT; 1487 1488 return kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT, 1489 (char *)&user_timeout, sizeof(user_timeout)); 1490} 1491 1492static void o2net_initialize_handshake(void) 1493{ 1494 o2net_hand->o2hb_heartbeat_timeout_ms = cpu_to_be32( 1495 O2HB_MAX_WRITE_TIMEOUT_MS); 1496 o2net_hand->o2net_idle_timeout_ms = cpu_to_be32(o2net_idle_timeout()); 1497 o2net_hand->o2net_keepalive_delay_ms = cpu_to_be32( 1498 o2net_keepalive_delay()); 1499 o2net_hand->o2net_reconnect_delay_ms = cpu_to_be32( 1500 o2net_reconnect_delay()); 1501} 1502 1503/* ------------------------------------------------------------ */ 1504 1505/* called when a connect completes and after a sock is accepted. the 1506 * rx path will see the response and mark the sc valid */ 1507static void o2net_sc_connect_completed(struct work_struct *work) 1508{ 1509 struct o2net_sock_container *sc = 1510 container_of(work, struct o2net_sock_container, 1511 sc_connect_work); 1512 1513 mlog(ML_MSG, "sc sending handshake with ver %llu id %llx\n", 1514 (unsigned long long)O2NET_PROTOCOL_VERSION, 1515 (unsigned long long)be64_to_cpu(o2net_hand->connector_id)); 1516 1517 o2net_initialize_handshake(); 1518 o2net_sendpage(sc, o2net_hand, sizeof(*o2net_hand)); 1519 sc_put(sc); 1520} 1521 1522/* this is called as a work_struct func. */ 1523static void o2net_sc_send_keep_req(struct work_struct *work) 1524{ 1525 struct o2net_sock_container *sc = 1526 container_of(work, struct o2net_sock_container, 1527 sc_keepalive_work.work); 1528 1529 o2net_sendpage(sc, o2net_keep_req, sizeof(*o2net_keep_req)); 1530 sc_put(sc); 1531} 1532 1533/* socket shutdown does a del_timer_sync against this as it tears down. 1534 * we can't start this timer until we've got to the point in sc buildup 1535 * where shutdown is going to be involved */ 1536static void o2net_idle_timer(unsigned long data) 1537{ 1538 struct o2net_sock_container *sc = (struct o2net_sock_container *)data; 1539 struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); 1540#ifdef CONFIG_DEBUG_FS 1541 unsigned long msecs = ktime_to_ms(ktime_get()) - 1542 ktime_to_ms(sc->sc_tv_timer); 1543#else 1544 unsigned long msecs = o2net_idle_timeout(); 1545#endif 1546 1547 printk(KERN_NOTICE "o2net: Connection to " SC_NODEF_FMT " has been " 1548 "idle for %lu.%lu secs.\n", 1549 SC_NODEF_ARGS(sc), msecs / 1000, msecs % 1000); 1550 1551 /* idle timerout happen, don't shutdown the connection, but 1552 * make fence decision. Maybe the connection can recover before 1553 * the decision is made. 1554 */ 1555 atomic_set(&nn->nn_timeout, 1); 1556 o2quo_conn_err(o2net_num_from_nn(nn)); 1557 queue_delayed_work(o2net_wq, &nn->nn_still_up, 1558 msecs_to_jiffies(O2NET_QUORUM_DELAY_MS)); 1559 1560 o2net_sc_reset_idle_timer(sc); 1561 1562} 1563 1564static void o2net_sc_reset_idle_timer(struct o2net_sock_container *sc) 1565{ 1566 o2net_sc_cancel_delayed_work(sc, &sc->sc_keepalive_work); 1567 o2net_sc_queue_delayed_work(sc, &sc->sc_keepalive_work, 1568 msecs_to_jiffies(o2net_keepalive_delay())); 1569 o2net_set_sock_timer(sc); 1570 mod_timer(&sc->sc_idle_timeout, 1571 jiffies + msecs_to_jiffies(o2net_idle_timeout())); 1572} 1573 1574static void o2net_sc_postpone_idle(struct o2net_sock_container *sc) 1575{ 1576 struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); 1577 1578 /* clear fence decision since the connection recover from timeout*/ 1579 if (atomic_read(&nn->nn_timeout)) { 1580 o2quo_conn_up(o2net_num_from_nn(nn)); 1581 cancel_delayed_work(&nn->nn_still_up); 1582 atomic_set(&nn->nn_timeout, 0); 1583 } 1584 1585 /* Only push out an existing timer */ 1586 if (timer_pending(&sc->sc_idle_timeout)) 1587 o2net_sc_reset_idle_timer(sc); 1588} 1589 1590/* this work func is kicked whenever a path sets the nn state which doesn't 1591 * have valid set. This includes seeing hb come up, losing a connection, 1592 * having a connect attempt fail, etc. This centralizes the logic which decides 1593 * if a connect attempt should be made or if we should give up and all future 1594 * transmit attempts should fail */ 1595static void o2net_start_connect(struct work_struct *work) 1596{ 1597 struct o2net_node *nn = 1598 container_of(work, struct o2net_node, nn_connect_work.work); 1599 struct o2net_sock_container *sc = NULL; 1600 struct o2nm_node *node = NULL, *mynode = NULL; 1601 struct socket *sock = NULL; 1602 struct sockaddr_in myaddr = {0, }, remoteaddr = {0, }; 1603 int ret = 0, stop; 1604 unsigned int timeout; 1605 unsigned int noio_flag; 1606 1607 /* 1608 * sock_create allocates the sock with GFP_KERNEL. We must set 1609 * per-process flag PF_MEMALLOC_NOIO so that all allocations done 1610 * by this process are done as if GFP_NOIO was specified. So we 1611 * are not reentering filesystem while doing memory reclaim. 1612 */ 1613 noio_flag = memalloc_noio_save(); 1614 /* if we're greater we initiate tx, otherwise we accept */ 1615 if (o2nm_this_node() <= o2net_num_from_nn(nn)) 1616 goto out; 1617 1618 /* watch for racing with tearing a node down */ 1619 node = o2nm_get_node_by_num(o2net_num_from_nn(nn)); 1620 if (node == NULL) { 1621 ret = 0; 1622 goto out; 1623 } 1624 1625 mynode = o2nm_get_node_by_num(o2nm_this_node()); 1626 if (mynode == NULL) { 1627 ret = 0; 1628 goto out; 1629 } 1630 1631 spin_lock(&nn->nn_lock); 1632 /* 1633 * see if we already have one pending or have given up. 1634 * For nn_timeout, it is set when we close the connection 1635 * because of the idle time out. So it means that we have 1636 * at least connected to that node successfully once, 1637 * now try to connect to it again. 1638 */ 1639 timeout = atomic_read(&nn->nn_timeout); 1640 stop = (nn->nn_sc || 1641 (nn->nn_persistent_error && 1642 (nn->nn_persistent_error != -ENOTCONN || timeout == 0))); 1643 spin_unlock(&nn->nn_lock); 1644 if (stop) 1645 goto out; 1646 1647 nn->nn_last_connect_attempt = jiffies; 1648 1649 sc = sc_alloc(node); 1650 if (sc == NULL) { 1651 mlog(0, "couldn't allocate sc\n"); 1652 ret = -ENOMEM; 1653 goto out; 1654 } 1655 1656 ret = sock_create(PF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); 1657 if (ret < 0) { 1658 mlog(0, "can't create socket: %d\n", ret); 1659 goto out; 1660 } 1661 sc->sc_sock = sock; /* freed by sc_kref_release */ 1662 1663 sock->sk->sk_allocation = GFP_ATOMIC; 1664 1665 myaddr.sin_family = AF_INET; 1666 myaddr.sin_addr.s_addr = mynode->nd_ipv4_address; 1667 myaddr.sin_port = htons(0); /* any port */ 1668 1669 ret = sock->ops->bind(sock, (struct sockaddr *)&myaddr, 1670 sizeof(myaddr)); 1671 if (ret) { 1672 mlog(ML_ERROR, "bind failed with %d at address %pI4\n", 1673 ret, &mynode->nd_ipv4_address); 1674 goto out; 1675 } 1676 1677 ret = o2net_set_nodelay(sc->sc_sock); 1678 if (ret) { 1679 mlog(ML_ERROR, "setting TCP_NODELAY failed with %d\n", ret); 1680 goto out; 1681 } 1682 1683 ret = o2net_set_usertimeout(sock); 1684 if (ret) { 1685 mlog(ML_ERROR, "set TCP_USER_TIMEOUT failed with %d\n", ret); 1686 goto out; 1687 } 1688 1689 o2net_register_callbacks(sc->sc_sock->sk, sc); 1690 1691 spin_lock(&nn->nn_lock); 1692 /* handshake completion will set nn->nn_sc_valid */ 1693 o2net_set_nn_state(nn, sc, 0, 0); 1694 spin_unlock(&nn->nn_lock); 1695 1696 remoteaddr.sin_family = AF_INET; 1697 remoteaddr.sin_addr.s_addr = node->nd_ipv4_address; 1698 remoteaddr.sin_port = node->nd_ipv4_port; 1699 1700 ret = sc->sc_sock->ops->connect(sc->sc_sock, 1701 (struct sockaddr *)&remoteaddr, 1702 sizeof(remoteaddr), 1703 O_NONBLOCK); 1704 if (ret == -EINPROGRESS) 1705 ret = 0; 1706 1707out: 1708 if (ret && sc) { 1709 printk(KERN_NOTICE "o2net: Connect attempt to " SC_NODEF_FMT 1710 " failed with errno %d\n", SC_NODEF_ARGS(sc), ret); 1711 /* 0 err so that another will be queued and attempted 1712 * from set_nn_state */ 1713 o2net_ensure_shutdown(nn, sc, 0); 1714 } 1715 if (sc) 1716 sc_put(sc); 1717 if (node) 1718 o2nm_node_put(node); 1719 if (mynode) 1720 o2nm_node_put(mynode); 1721 1722 memalloc_noio_restore(noio_flag); 1723 return; 1724} 1725 1726static void o2net_connect_expired(struct work_struct *work) 1727{ 1728 struct o2net_node *nn = 1729 container_of(work, struct o2net_node, nn_connect_expired.work); 1730 1731 spin_lock(&nn->nn_lock); 1732 if (!nn->nn_sc_valid) { 1733 printk(KERN_NOTICE "o2net: No connection established with " 1734 "node %u after %u.%u seconds, check network and" 1735 " cluster configuration.\n", 1736 o2net_num_from_nn(nn), 1737 o2net_idle_timeout() / 1000, 1738 o2net_idle_timeout() % 1000); 1739 1740 o2net_set_nn_state(nn, NULL, 0, 0); 1741 } 1742 spin_unlock(&nn->nn_lock); 1743} 1744 1745static void o2net_still_up(struct work_struct *work) 1746{ 1747 struct o2net_node *nn = 1748 container_of(work, struct o2net_node, nn_still_up.work); 1749 1750 o2quo_hb_still_up(o2net_num_from_nn(nn)); 1751} 1752 1753/* ------------------------------------------------------------ */ 1754 1755void o2net_disconnect_node(struct o2nm_node *node) 1756{ 1757 struct o2net_node *nn = o2net_nn_from_num(node->nd_num); 1758 1759 /* don't reconnect until it's heartbeating again */ 1760 spin_lock(&nn->nn_lock); 1761 atomic_set(&nn->nn_timeout, 0); 1762 o2net_set_nn_state(nn, NULL, 0, -ENOTCONN); 1763 spin_unlock(&nn->nn_lock); 1764 1765 if (o2net_wq) { 1766 cancel_delayed_work(&nn->nn_connect_expired); 1767 cancel_delayed_work(&nn->nn_connect_work); 1768 cancel_delayed_work(&nn->nn_still_up); 1769 flush_workqueue(o2net_wq); 1770 } 1771} 1772 1773static void o2net_hb_node_down_cb(struct o2nm_node *node, int node_num, 1774 void *data) 1775{ 1776 o2quo_hb_down(node_num); 1777 1778 if (!node) 1779 return; 1780 1781 if (node_num != o2nm_this_node()) 1782 o2net_disconnect_node(node); 1783 1784 BUG_ON(atomic_read(&o2net_connected_peers) < 0); 1785} 1786 1787static void o2net_hb_node_up_cb(struct o2nm_node *node, int node_num, 1788 void *data) 1789{ 1790 struct o2net_node *nn = o2net_nn_from_num(node_num); 1791 1792 o2quo_hb_up(node_num); 1793 1794 BUG_ON(!node); 1795 1796 /* ensure an immediate connect attempt */ 1797 nn->nn_last_connect_attempt = jiffies - 1798 (msecs_to_jiffies(o2net_reconnect_delay()) + 1); 1799 1800 if (node_num != o2nm_this_node()) { 1801 /* believe it or not, accept and node hearbeating testing 1802 * can succeed for this node before we got here.. so 1803 * only use set_nn_state to clear the persistent error 1804 * if that hasn't already happened */ 1805 spin_lock(&nn->nn_lock); 1806 atomic_set(&nn->nn_timeout, 0); 1807 if (nn->nn_persistent_error) 1808 o2net_set_nn_state(nn, NULL, 0, 0); 1809 spin_unlock(&nn->nn_lock); 1810 } 1811} 1812 1813void o2net_unregister_hb_callbacks(void) 1814{ 1815 o2hb_unregister_callback(NULL, &o2net_hb_up); 1816 o2hb_unregister_callback(NULL, &o2net_hb_down); 1817} 1818 1819int o2net_register_hb_callbacks(void) 1820{ 1821 int ret; 1822 1823 o2hb_setup_callback(&o2net_hb_down, O2HB_NODE_DOWN_CB, 1824 o2net_hb_node_down_cb, NULL, O2NET_HB_PRI); 1825 o2hb_setup_callback(&o2net_hb_up, O2HB_NODE_UP_CB, 1826 o2net_hb_node_up_cb, NULL, O2NET_HB_PRI); 1827 1828 ret = o2hb_register_callback(NULL, &o2net_hb_up); 1829 if (ret == 0) 1830 ret = o2hb_register_callback(NULL, &o2net_hb_down); 1831 1832 if (ret) 1833 o2net_unregister_hb_callbacks(); 1834 1835 return ret; 1836} 1837 1838/* ------------------------------------------------------------ */ 1839 1840static int o2net_accept_one(struct socket *sock, int *more) 1841{ 1842 int ret, slen; 1843 struct sockaddr_in sin; 1844 struct socket *new_sock = NULL; 1845 struct o2nm_node *node = NULL; 1846 struct o2nm_node *local_node = NULL; 1847 struct o2net_sock_container *sc = NULL; 1848 struct o2net_node *nn; 1849 unsigned int noio_flag; 1850 1851 /* 1852 * sock_create_lite allocates the sock with GFP_KERNEL. We must set 1853 * per-process flag PF_MEMALLOC_NOIO so that all allocations done 1854 * by this process are done as if GFP_NOIO was specified. So we 1855 * are not reentering filesystem while doing memory reclaim. 1856 */ 1857 noio_flag = memalloc_noio_save(); 1858 1859 BUG_ON(sock == NULL); 1860 *more = 0; 1861 ret = sock_create_lite(sock->sk->sk_family, sock->sk->sk_type, 1862 sock->sk->sk_protocol, &new_sock); 1863 if (ret) 1864 goto out; 1865 1866 new_sock->type = sock->type; 1867 new_sock->ops = sock->ops; 1868 ret = sock->ops->accept(sock, new_sock, O_NONBLOCK); 1869 if (ret < 0) 1870 goto out; 1871 1872 *more = 1; 1873 new_sock->sk->sk_allocation = GFP_ATOMIC; 1874 1875 ret = o2net_set_nodelay(new_sock); 1876 if (ret) { 1877 mlog(ML_ERROR, "setting TCP_NODELAY failed with %d\n", ret); 1878 goto out; 1879 } 1880 1881 ret = o2net_set_usertimeout(new_sock); 1882 if (ret) { 1883 mlog(ML_ERROR, "set TCP_USER_TIMEOUT failed with %d\n", ret); 1884 goto out; 1885 } 1886 1887 slen = sizeof(sin); 1888 ret = new_sock->ops->getname(new_sock, (struct sockaddr *) &sin, 1889 &slen, 1); 1890 if (ret < 0) 1891 goto out; 1892 1893 node = o2nm_get_node_by_ip(sin.sin_addr.s_addr); 1894 if (node == NULL) { 1895 printk(KERN_NOTICE "o2net: Attempt to connect from unknown " 1896 "node at %pI4:%d\n", &sin.sin_addr.s_addr, 1897 ntohs(sin.sin_port)); 1898 ret = -EINVAL; 1899 goto out; 1900 } 1901 1902 if (o2nm_this_node() >= node->nd_num) { 1903 local_node = o2nm_get_node_by_num(o2nm_this_node()); 1904 if (local_node) 1905 printk(KERN_NOTICE "o2net: Unexpected connect attempt " 1906 "seen at node '%s' (%u, %pI4:%d) from " 1907 "node '%s' (%u, %pI4:%d)\n", 1908 local_node->nd_name, local_node->nd_num, 1909 &(local_node->nd_ipv4_address), 1910 ntohs(local_node->nd_ipv4_port), 1911 node->nd_name, 1912 node->nd_num, &sin.sin_addr.s_addr, 1913 ntohs(sin.sin_port)); 1914 ret = -EINVAL; 1915 goto out; 1916 } 1917 1918 /* this happens all the time when the other node sees our heartbeat 1919 * and tries to connect before we see their heartbeat */ 1920 if (!o2hb_check_node_heartbeating_from_callback(node->nd_num)) { 1921 mlog(ML_CONN, "attempt to connect from node '%s' at " 1922 "%pI4:%d but it isn't heartbeating\n", 1923 node->nd_name, &sin.sin_addr.s_addr, 1924 ntohs(sin.sin_port)); 1925 ret = -EINVAL; 1926 goto out; 1927 } 1928 1929 nn = o2net_nn_from_num(node->nd_num); 1930 1931 spin_lock(&nn->nn_lock); 1932 if (nn->nn_sc) 1933 ret = -EBUSY; 1934 else 1935 ret = 0; 1936 spin_unlock(&nn->nn_lock); 1937 if (ret) { 1938 printk(KERN_NOTICE "o2net: Attempt to connect from node '%s' " 1939 "at %pI4:%d but it already has an open connection\n", 1940 node->nd_name, &sin.sin_addr.s_addr, 1941 ntohs(sin.sin_port)); 1942 goto out; 1943 } 1944 1945 sc = sc_alloc(node); 1946 if (sc == NULL) { 1947 ret = -ENOMEM; 1948 goto out; 1949 } 1950 1951 sc->sc_sock = new_sock; 1952 new_sock = NULL; 1953 1954 spin_lock(&nn->nn_lock); 1955 atomic_set(&nn->nn_timeout, 0); 1956 o2net_set_nn_state(nn, sc, 0, 0); 1957 spin_unlock(&nn->nn_lock); 1958 1959 o2net_register_callbacks(sc->sc_sock->sk, sc); 1960 o2net_sc_queue_work(sc, &sc->sc_rx_work); 1961 1962 o2net_initialize_handshake(); 1963 o2net_sendpage(sc, o2net_hand, sizeof(*o2net_hand)); 1964 1965out: 1966 if (new_sock) 1967 sock_release(new_sock); 1968 if (node) 1969 o2nm_node_put(node); 1970 if (local_node) 1971 o2nm_node_put(local_node); 1972 if (sc) 1973 sc_put(sc); 1974 1975 memalloc_noio_restore(noio_flag); 1976 return ret; 1977} 1978 1979/* 1980 * This function is invoked in response to one or more 1981 * pending accepts at softIRQ level. We must drain the 1982 * entire que before returning. 1983 */ 1984 1985static void o2net_accept_many(struct work_struct *work) 1986{ 1987 struct socket *sock = o2net_listen_sock; 1988 int more; 1989 int err; 1990 1991 /* 1992 * It is critical to note that due to interrupt moderation 1993 * at the network driver level, we can't assume to get a 1994 * softIRQ for every single conn since tcp SYN packets 1995 * can arrive back-to-back, and therefore many pending 1996 * accepts may result in just 1 softIRQ. If we terminate 1997 * the o2net_accept_one() loop upon seeing an err, what happens 1998 * to the rest of the conns in the queue? If no new SYN 1999 * arrives for hours, no softIRQ will be delivered, 2000 * and the connections will just sit in the queue. 2001 */ 2002 2003 for (;;) { 2004 err = o2net_accept_one(sock, &more); 2005 if (!more) 2006 break; 2007 cond_resched(); 2008 } 2009} 2010 2011static void o2net_listen_data_ready(struct sock *sk) 2012{ 2013 void (*ready)(struct sock *sk); 2014 2015 read_lock(&sk->sk_callback_lock); 2016 ready = sk->sk_user_data; 2017 if (ready == NULL) { /* check for teardown race */ 2018 ready = sk->sk_data_ready; 2019 goto out; 2020 } 2021 2022 /* This callback may called twice when a new connection 2023 * is being established as a child socket inherits everything 2024 * from a parent LISTEN socket, including the data_ready cb of 2025 * the parent. This leads to a hazard. In o2net_accept_one() 2026 * we are still initializing the child socket but have not 2027 * changed the inherited data_ready callback yet when 2028 * data starts arriving. 2029 * We avoid this hazard by checking the state. 2030 * For the listening socket, the state will be TCP_LISTEN; for the new 2031 * socket, will be TCP_ESTABLISHED. Also, in this case, 2032 * sk->sk_user_data is not a valid function pointer. 2033 */ 2034 2035 if (sk->sk_state == TCP_LISTEN) { 2036 queue_work(o2net_wq, &o2net_listen_work); 2037 } else { 2038 ready = NULL; 2039 } 2040 2041out: 2042 read_unlock(&sk->sk_callback_lock); 2043 if (ready != NULL) 2044 ready(sk); 2045} 2046 2047static int o2net_open_listening_sock(__be32 addr, __be16 port) 2048{ 2049 struct socket *sock = NULL; 2050 int ret; 2051 struct sockaddr_in sin = { 2052 .sin_family = PF_INET, 2053 .sin_addr = { .s_addr = addr }, 2054 .sin_port = port, 2055 }; 2056 2057 ret = sock_create(PF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); 2058 if (ret < 0) { 2059 printk(KERN_ERR "o2net: Error %d while creating socket\n", ret); 2060 goto out; 2061 } 2062 2063 sock->sk->sk_allocation = GFP_ATOMIC; 2064 2065 write_lock_bh(&sock->sk->sk_callback_lock); 2066 sock->sk->sk_user_data = sock->sk->sk_data_ready; 2067 sock->sk->sk_data_ready = o2net_listen_data_ready; 2068 write_unlock_bh(&sock->sk->sk_callback_lock); 2069 2070 o2net_listen_sock = sock; 2071 INIT_WORK(&o2net_listen_work, o2net_accept_many); 2072 2073 sock->sk->sk_reuse = SK_CAN_REUSE; 2074 ret = sock->ops->bind(sock, (struct sockaddr *)&sin, sizeof(sin)); 2075 if (ret < 0) { 2076 printk(KERN_ERR "o2net: Error %d while binding socket at " 2077 "%pI4:%u\n", ret, &addr, ntohs(port)); 2078 goto out; 2079 } 2080 2081 ret = sock->ops->listen(sock, 64); 2082 if (ret < 0) 2083 printk(KERN_ERR "o2net: Error %d while listening on %pI4:%u\n", 2084 ret, &addr, ntohs(port)); 2085 2086out: 2087 if (ret) { 2088 o2net_listen_sock = NULL; 2089 if (sock) 2090 sock_release(sock); 2091 } 2092 return ret; 2093} 2094 2095/* 2096 * called from node manager when we should bring up our network listening 2097 * socket. node manager handles all the serialization to only call this 2098 * once and to match it with o2net_stop_listening(). note, 2099 * o2nm_this_node() doesn't work yet as we're being called while it 2100 * is being set up. 2101 */ 2102int o2net_start_listening(struct o2nm_node *node) 2103{ 2104 int ret = 0; 2105 2106 BUG_ON(o2net_wq != NULL); 2107 BUG_ON(o2net_listen_sock != NULL); 2108 2109 mlog(ML_KTHREAD, "starting o2net thread...\n"); 2110 o2net_wq = create_singlethread_workqueue("o2net"); 2111 if (o2net_wq == NULL) { 2112 mlog(ML_ERROR, "unable to launch o2net thread\n"); 2113 return -ENOMEM; /* ? */ 2114 } 2115 2116 ret = o2net_open_listening_sock(node->nd_ipv4_address, 2117 node->nd_ipv4_port); 2118 if (ret) { 2119 destroy_workqueue(o2net_wq); 2120 o2net_wq = NULL; 2121 } else 2122 o2quo_conn_up(node->nd_num); 2123 2124 return ret; 2125} 2126 2127/* again, o2nm_this_node() doesn't work here as we're involved in 2128 * tearing it down */ 2129void o2net_stop_listening(struct o2nm_node *node) 2130{ 2131 struct socket *sock = o2net_listen_sock; 2132 size_t i; 2133 2134 BUG_ON(o2net_wq == NULL); 2135 BUG_ON(o2net_listen_sock == NULL); 2136 2137 /* stop the listening socket from generating work */ 2138 write_lock_bh(&sock->sk->sk_callback_lock); 2139 sock->sk->sk_data_ready = sock->sk->sk_user_data; 2140 sock->sk->sk_user_data = NULL; 2141 write_unlock_bh(&sock->sk->sk_callback_lock); 2142 2143 for (i = 0; i < ARRAY_SIZE(o2net_nodes); i++) { 2144 struct o2nm_node *node = o2nm_get_node_by_num(i); 2145 if (node) { 2146 o2net_disconnect_node(node); 2147 o2nm_node_put(node); 2148 } 2149 } 2150 2151 /* finish all work and tear down the work queue */ 2152 mlog(ML_KTHREAD, "waiting for o2net thread to exit....\n"); 2153 destroy_workqueue(o2net_wq); 2154 o2net_wq = NULL; 2155 2156 sock_release(o2net_listen_sock); 2157 o2net_listen_sock = NULL; 2158 2159 o2quo_conn_err(node->nd_num); 2160} 2161 2162/* ------------------------------------------------------------ */ 2163 2164int o2net_init(void) 2165{ 2166 unsigned long i; 2167 2168 o2quo_init(); 2169 2170 if (o2net_debugfs_init()) 2171 goto out; 2172 2173 o2net_hand = kzalloc(sizeof(struct o2net_handshake), GFP_KERNEL); 2174 o2net_keep_req = kzalloc(sizeof(struct o2net_msg), GFP_KERNEL); 2175 o2net_keep_resp = kzalloc(sizeof(struct o2net_msg), GFP_KERNEL); 2176 if (!o2net_hand || !o2net_keep_req || !o2net_keep_resp) 2177 goto out; 2178 2179 o2net_hand->protocol_version = cpu_to_be64(O2NET_PROTOCOL_VERSION); 2180 o2net_hand->connector_id = cpu_to_be64(1); 2181 2182 o2net_keep_req->magic = cpu_to_be16(O2NET_MSG_KEEP_REQ_MAGIC); 2183 o2net_keep_resp->magic = cpu_to_be16(O2NET_MSG_KEEP_RESP_MAGIC); 2184 2185 for (i = 0; i < ARRAY_SIZE(o2net_nodes); i++) { 2186 struct o2net_node *nn = o2net_nn_from_num(i); 2187 2188 atomic_set(&nn->nn_timeout, 0); 2189 spin_lock_init(&nn->nn_lock); 2190 INIT_DELAYED_WORK(&nn->nn_connect_work, o2net_start_connect); 2191 INIT_DELAYED_WORK(&nn->nn_connect_expired, 2192 o2net_connect_expired); 2193 INIT_DELAYED_WORK(&nn->nn_still_up, o2net_still_up); 2194 /* until we see hb from a node we'll return einval */ 2195 nn->nn_persistent_error = -ENOTCONN; 2196 init_waitqueue_head(&nn->nn_sc_wq); 2197 idr_init(&nn->nn_status_idr); 2198 INIT_LIST_HEAD(&nn->nn_status_list); 2199 } 2200 2201 return 0; 2202 2203out: 2204 kfree(o2net_hand); 2205 kfree(o2net_keep_req); 2206 kfree(o2net_keep_resp); 2207 2208 o2quo_exit(); 2209 return -ENOMEM; 2210} 2211 2212void o2net_exit(void) 2213{ 2214 o2quo_exit(); 2215 kfree(o2net_hand); 2216 kfree(o2net_keep_req); 2217 kfree(o2net_keep_resp); 2218 o2net_debugfs_exit(); 2219} 2220