diff --git a/homa_rpc.c b/homa_rpc.c index bdbe8752..6388e952 100644 --- a/homa_rpc.c +++ b/homa_rpc.c @@ -540,6 +540,14 @@ int homa_rpc_reap(struct homa_sock *hsk) continue; } + /* Release the RPC's share of the socket's resources while + * the RPC is still on dead_rpcs and the socket is locked: + * once neither holds, homa_sock_destroy may free them. + */ + homa_pool_release(rpc); + WARN_ON(refcount_sub_and_test(rpc->msgout.frag_bytes, + &hsk->sock.sk_wmem_alloc)); + rpcs[num_rpcs] = rpc; num_rpcs++; list_del(&rpc->dead_links); @@ -548,10 +556,16 @@ int homa_rpc_reap(struct homa_sock *hsk) } checked_all_rpcs = true; - /* Free all of the collected resources; release the socket lock - * while doing this. + /* Free the rest of the collected resources (none of them belong + * to the socket); release the socket lock while doing this. */ release: + /* Wake up writers only if this call released memory: a call that + * starts after homa_sock_destroy finds no RPCs, and must not touch + * sk_wq. + */ + if (num_rpcs > 0) + homa_sock_wakeup_wmem(hsk); homa_sock_unlock(hsk); for (i = 0; i < num_rpcs; i++) { IF_NO_STRIP(int tx_left); @@ -599,11 +613,8 @@ int homa_rpc_reap(struct homa_sock *hsk) homa_route_release(rpc->route); rpc->route = NULL; } - homa_pool_release(rpc); homa_tx_pool_free(hsk->homa, rpc->msgout.frag_nid, rpc->msgout.num_frags, rpc->msgout.frags); - WARN_ON(refcount_sub_and_test(rpc->msgout.frag_bytes, - &hsk->sock.sk_wmem_alloc)); if (rpc->msgout.frags != &rpc->msgout.frag) kvfree(rpc->msgout.frags); tt_record2("homa_rpc_reap finished reaping id %d, port %d", @@ -634,7 +645,6 @@ int homa_rpc_reap(struct homa_sock *hsk) rpc->magic = 0; kfree(rpc); } - homa_sock_wakeup_wmem(hsk); tt_record3("reaped %d rpcs; %d dead frags remain for port %d", num_rpcs, hsk->dead_frags, hsk->port); return !checked_all_rpcs; diff --git a/homa_sock.c b/homa_sock.c index 85c2c9b9..ed794f66 100644 --- a/homa_sock.c +++ b/homa_sock.c @@ -402,8 +402,8 @@ void homa_sock_shutdown(struct homa_sock *hsk) /** * homa_sock_destroy() - Release all of the internal resources associated - * with a socket; is invoked at time when that is safe (i.e., all references - * on the socket have been dropped). + * with a socket. Other threads (e.g. homa_timer) may still hold references + * to the socket and be reaping its RPCs. * @sk: Socket to destroy. */ void homa_sock_destroy(struct sock *sk) @@ -436,6 +436,16 @@ void homa_sock_destroy(struct sock *sk) #endif /* See strip.py */ } + /* Another thread (e.g. homa_timer) may still be in homa_rpc_reap, + * releasing the pool space and sk_wmem_alloc of the last RPCs that + * it took off dead_rpcs and waking up writers through sk_wq, which + * sk_common_release sets to NULL after this function returns. It + * does all of that before it unlocks the socket, so wait for the + * lock. + */ + homa_sock_lock(hsk); + homa_sock_unlock(hsk); + WARN_ON_ONCE(refcount_read(&hsk->sock.sk_wmem_alloc) != 1); #ifdef __UNIT_TEST__ { diff --git a/test/unit_homa_rpc.c b/test/unit_homa_rpc.c index d201be6e..fbfa3080 100644 --- a/test/unit_homa_rpc.c +++ b/test/unit_homa_rpc.c @@ -1002,6 +1002,55 @@ TEST_F(homa_rpc, homa_rpc_reap__release_bpages) homa_rpc_reap(&self->hsk); EXPECT_EQ(0, pool_size - homa_pool_avail_bytes(pool)); } +static struct homa_sock *hook_hsk; +static int hook_early_unlocks; +static void reap_unlock_hook(char *id) +{ + struct homa_pool *pool = hook_hsk->buffer_pool; + + if (strcmp(id, "unlock") != 0 || !list_empty(&hook_hsk->dead_rpcs)) + return; + if (refcount_read(&hook_hsk->sock.sk_wmem_alloc) != 1 || + homa_pool_avail_bytes(pool) != pool->num_bpages * HOMA_BPAGE_SIZE || + test_bit(HOMA_SOCK_NOSPACE, &hook_hsk->flags)) + hook_early_unlocks++; +} +TEST_F(homa_rpc, homa_rpc_reap__release_socket_resources_before_unlock) +{ + struct homa_rpc *crpc; + + crpc = unit_client_rpc(&self->hsk, UNIT_RCVD_ONE_PKT, self->client_ip, + self->server_ip, 4000, 98, 1000, 150000); + ASSERT_NE(NULL, crpc); + homa_rpc_end(crpc); + set_bit(HOMA_SOCK_NOSPACE, &self->hsk.flags); + hook_hsk = &self->hsk; + hook_early_unlocks = 0; + unit_hook_register(reap_unlock_hook); + + homa_rpc_reap(&self->hsk); + unit_hook_clear(); + EXPECT_EQ(0, hook_early_unlocks); + EXPECT_EQ(1, refcount_read(&self->hsk.sock.sk_wmem_alloc)); +} +TEST_F(homa_rpc, homa_rpc_reap__no_wakeup_if_no_rpcs_reaped) +{ + struct homa_rpc *crpc; + + crpc = unit_client_rpc(&self->hsk, UNIT_RCVD_ONE_PKT, self->client_ip, + self->server_ip, 4000, 98, 1000, 2000); + ASSERT_NE(NULL, crpc); + homa_rpc_end(crpc); + set_bit(HOMA_SOCK_NOSPACE, &self->hsk.flags); + homa_rpc_hold(crpc); + + homa_rpc_reap(&self->hsk); + EXPECT_EQ(1, test_bit(HOMA_SOCK_NOSPACE, &self->hsk.flags)); + + homa_rpc_put(crpc); + homa_rpc_reap(&self->hsk); + EXPECT_EQ(0, test_bit(HOMA_SOCK_NOSPACE, &self->hsk.flags)); +} TEST_F(homa_rpc, homa_rpc_reap__skb_memory_accounting) { struct homa_rpc *crpc1 = unit_client_rpc(&self->hsk,