diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 26c7a1b9dc..d910c7932b 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -75,12 +75,25 @@ static const uint32_t HELLO_ACK_URMA_OK = 0x1; static const size_t IOBUF_BLOCK_HEADER_LEN = sizeof(butil::IOBuf::Block); // ---- Globals: prepared jetty pool + poller groups ---- -struct PreparedJetty { - UrmaResource* res; -}; static butil::Mutex g_prepared_mutex; -static UrmaResource* g_prepared_list = nullptr; // singly-linked -static int g_prepared_cnt = 0; +using PreparedResources = std::vector>; + +static PreparedResources& PreparedJettyResources() { + // The URMA context is kept alive until process exit. Release this cache + // explicitly during initialization rollback, before deleting the context. + static auto* resources = new PreparedResources; + return *resources; +} + +static uint16_t QueueDepth(int depth) { + return static_cast(std::max(16, std::min(4096, depth))); +} + +static LocalJettyConfig MakeLocalJettyConfig( + urma_context_t* ctx, uint16_t sq_depth, uint16_t rq_depth) { + return {ctx, sq_depth, rq_depth, static_cast(GetUrmaMaxSge()), + GetUrmaJettyPriority(), FLAGS_urma_use_polling}; +} static int PreparedJettyCount() { const int requested = @@ -114,31 +127,6 @@ static int PreparedJettyCount() { std::vector UrmaEndpoint::_poller_groups; -// ============================================================================ -// UrmaResource lifecycle. -// ============================================================================ - -UrmaResource::~UrmaResource() { - if (remote_jetty) { - urma_unimport_jetty(remote_jetty); - } - if (remote_seg) { - urma_unimport_seg(remote_seg); - } - if (jetty) { - urma_delete_jetty(jetty); - } - if (jfr) { - urma_delete_jfr(jfr); - } - if (jfc) { - urma_delete_jfc(jfc); - } - if (jfce) { - urma_delete_jfce(jfce); - } -} - // ============================================================================ // Constructor / destructor / Reset. // ============================================================================ @@ -146,12 +134,9 @@ UrmaResource::~UrmaResource() { UrmaEndpoint::UrmaEndpoint(Socket* s) : _socket(s), _state(UNINIT), - _handshake_version(0), - _resource(nullptr) { - _sq_size = static_cast( - std::max(16, std::min(4096, static_cast(FLAGS_urma_sq_size)))); - _rq_size = static_cast( - std::max(16, std::min(4096, static_cast(FLAGS_urma_rq_size)))); + _handshake_version(0) { + _sq_size = QueueDepth(FLAGS_urma_sq_size); + _rq_size = QueueDepth(FLAGS_urma_rq_size); _read_butex = bthread::butex_create_checked>(); _read_butex->store(0, butil::memory_order_relaxed); _input_processor.Init(s, InputMessengerProcessor::STREAM_URMA_JETTY); @@ -396,72 +381,18 @@ int UrmaEndpoint::AllocateResources() { return -1; } - _resource = new (std::nothrow) UrmaResource(); - if (!_resource) { - return -1; - } - - // Try the prepared pool first (sized sq/rq match). - if (_sq_size <= static_cast(FLAGS_urma_sq_size) && - _rq_size <= static_cast(FLAGS_urma_rq_size)) { + const auto config = MakeLocalJettyConfig(ctx, _sq_size, _rq_size); + { BAIDU_SCOPED_LOCK(g_prepared_mutex); - if (g_prepared_list) { - UrmaResource* next = g_prepared_list->next; - delete _resource; - _resource = g_prepared_list; - g_prepared_list = next; - _resource->next = nullptr; - --g_prepared_cnt; + auto& prepared = PreparedJettyResources(); + if (!prepared.empty() && prepared.back()->config == config) { + _resource = std::move(prepared.back()); + prepared.pop_back(); } } - - if (!_resource->jfc) { - // The SDK requires every JFC to reference a JFCE. Polling mode does - // not arm or consume it, but still supplies the required object. - _resource->jfce = urma_create_jfce(ctx); - if (!_resource->jfce || - (!FLAGS_urma_use_polling && _resource->jfce->fd < 0)) { - LOG(ERROR) << "Fail to create a usable URMA JFCE"; - errno = ENODEV; - return -1; - } - - urma_jfc_cfg_t jfc_cfg{}; - jfc_cfg.depth = static_cast(_sq_size + _rq_size); - jfc_cfg.jfce = _resource->jfce; - _resource->jfc = urma_create_jfc(ctx, &jfc_cfg); - if (!_resource->jfc) { - PLOG(ERROR) << "urma_create_jfc"; - return -1; - } - - urma_jfr_cfg_t jfr_cfg{}; - jfr_cfg.depth = static_cast(_rq_size); - jfr_cfg.trans_mode = URMA_TM_RM; - jfr_cfg.max_sge = 1; - jfr_cfg.min_rnr_timer = URMA_TYPICAL_MIN_RNR_TIMER; - jfr_cfg.jfc = _resource->jfc; - _resource->jfr = urma_create_jfr(ctx, &jfr_cfg); - if (!_resource->jfr) { - PLOG(ERROR) << "urma_create_jfr"; - return -1; - } - - urma_jetty_cfg_t jetty_cfg{}; - jetty_cfg.flag.bs.share_jfr = 1; - jetty_cfg.jfs_cfg.depth = static_cast(_sq_size); - jetty_cfg.jfs_cfg.trans_mode = URMA_TM_RM; - jetty_cfg.jfs_cfg.priority = GetUrmaJettyPriority(); - jetty_cfg.jfs_cfg.max_sge = - static_cast(GetUrmaMaxSge()); - jetty_cfg.jfs_cfg.rnr_retry = URMA_TYPICAL_RNR_RETRY; - jetty_cfg.jfs_cfg.err_timeout = URMA_TYPICAL_ERR_TIMEOUT; - jetty_cfg.jfs_cfg.jfc = _resource->jfc; - jetty_cfg.shared.jfr = _resource->jfr; - jetty_cfg.shared.jfc = _resource->jfc; - _resource->jetty = urma_create_jetty(ctx, &jetty_cfg); - if (!_resource->jetty) { - PLOG(ERROR) << "urma_create_jetty"; + if (!_resource) { + _resource = LocalJettyResources::Create(config); + if (!_resource) { return -1; } } @@ -471,18 +402,15 @@ int UrmaEndpoint::AllocateResources() { _rbuf_data.resize(_rq_size, nullptr); // Wrap the JFCE fd in a brpc Socket so PollCq is driven by epoll. - if (!FLAGS_urma_use_polling) { - if (!_resource->jfce || _resource->jfce->fd < 0) { - LOG(ERROR) << "Prepared URMA resource has no usable JFCE"; - errno = ENODEV; - return -1; - } + if (!_resource->config.use_polling) { if (ReqNotifyCq() != 0) { return -1; } SocketOptions options; options.user = this; options.keytable_pool = _socket->keytable_pool(); + // liburma owns the JFCE fd. Detach it from the CQ socket before + // releasing the local resources. options.fd = _resource->jfce->fd; options.on_edge_triggered_events = PollCq; if (Socket::Create(options, &_cq_sid) < 0) { @@ -509,7 +437,7 @@ void UrmaEndpoint::DeallocateResources() { return; } - if (FLAGS_urma_use_polling) { + if (_resource->config.use_polling) { PollerRemoveCqSid(); } @@ -530,8 +458,8 @@ void UrmaEndpoint::DeallocateResources() { // Reusing a Jetty requires a driver-supported RESET plus a complete JFC // drain. Until that lifecycle is implemented, prepared resources are // one-shot: they accelerate connection setup but are destroyed on close. - delete _resource; - _resource = nullptr; + _peer.Reset(); + _resource.reset(); } // ============================================================================ @@ -559,8 +487,8 @@ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { seg_flag.bs.cacheable = URMA_NON_CACHEABLE; seg_flag.bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE | URMA_ACCESS_ATOMIC; seg_flag.bs.mapping = URMA_SEG_NOMAP; - _resource->remote_seg = urma_import_seg(ctx, &peer_seg, &seg_token, 0, seg_flag); - if (!_resource->remote_seg) { + _peer.segment = urma_import_seg(ctx, &peer_seg, &seg_token, 0, seg_flag); + if (!_peer.segment) { PLOG(ERROR) << "urma_import_seg failed"; return -1; } @@ -591,7 +519,7 @@ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { bonding_remote.base = remote; bonding_remote.base.flag.bs.has_drv_ext = 1; bonding_remote.jetty = _resource->jetty; - _resource->remote_jetty = + _peer.jetty = urma_import_jetty(ctx, &bonding_remote.base, &token); #else LOG(ERROR) << "Bonding remote jetty import requires provider header " @@ -599,9 +527,9 @@ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { errno = ENOTSUP; #endif } else { - _resource->remote_jetty = urma_import_jetty(ctx, &remote, &token); + _peer.jetty = urma_import_jetty(ctx, &remote, &token); } - if (!_resource->remote_jetty) { + if (!_peer.jetty) { if (errno == 0) { errno = EIO; } @@ -673,7 +601,7 @@ class UrmaIOBuf : private butil::IOBuf { }; ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { - if (!_resource || !_resource->jetty || !_resource->remote_jetty) { + if (!_resource || !_resource->jetty || !_peer.jetty) { errno = ENOTCONN; return -1; } @@ -734,13 +662,13 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { // produce asymmetric completions with the bonding provider. wr.opcode = URMA_OPC_SEND; wr.flag.bs.complete_enable = 1; - wr.tjetty = _resource->remote_jetty; + wr.tjetty = _peer.jetty; wr.send.src = sg; wr.user_ctx = 1; urma_jfs_wr_t* bad_wr = nullptr; const uint16_t sq_slot = _sq_current; const uint32_t local_jetty_id = _resource->jetty->jetty_id.id; - const uint32_t remote_jetty_id = _resource->remote_jetty->id.id; + const uint32_t remote_jetty_id = _peer.jetty->id.id; // Reserve both credits before making the WR visible to the provider. // In polling mode a completion (and even the peer's receive-credit @@ -867,7 +795,7 @@ int UrmaEndpoint::SendImm(uint32_t imm) { if (imm == 0) { return 0; } - if (!_resource || !_resource->jetty || !_resource->remote_jetty) { + if (!_resource || !_resource->jetty || !_peer.jetty) { errno = ENOTCONN; return -1; } @@ -882,7 +810,7 @@ int UrmaEndpoint::SendImm(uint32_t imm) { wr.opcode = URMA_OPC_SEND_IMM; wr.flag.bs.complete_enable = 1; wr.flag.bs.solicited_enable = 1; - wr.tjetty = _resource->remote_jetty; + wr.tjetty = _peer.jetty; wr.send.imm_data = imm; wr.user_ctx = 0; // 0 == pure ack (HandleCompletion reuses budget). urma_jfs_wr_t* bad = nullptr; @@ -905,7 +833,7 @@ int UrmaEndpoint::SendImm(uint32_t imm) { << " local_jetty_id=" << _resource->jetty->jetty_id.id << " remote_jetty_id=" - << _resource->remote_jetty->id.id + << _peer.jetty->id.id << " state=" << GetStateStr() << " on " << _socket->description(); errno = status; @@ -1092,7 +1020,7 @@ void UrmaEndpoint::PollCq(Socket* m) { return; } - const bool event_mode = !FLAGS_urma_use_polling; + const bool event_mode = !ep->_resource->config.use_polling; int progress = Socket::PROGRESS_INIT; while (true) { urma_jfc_t* event_jfc = nullptr; @@ -1766,78 +1694,35 @@ int UrmaEndpoint::GlobalInitialize() { } _poller_groups = std::vector(ntags); } - if (g_prepared_cnt > 0) { - return 0; - } urma_context_t* ctx = GetUrmaContext(); if (!ctx) { return 0; } + + auto& prepared = PreparedJettyResources(); + if (!prepared.empty()) { + return 0; + } + const auto config = MakeLocalJettyConfig( + ctx, QueueDepth(FLAGS_urma_sq_size), QueueDepth(FLAGS_urma_rq_size)); const int prepared_jetty_count = PreparedJettyCount(); for (int i = 0; i < prepared_jetty_count; ++i) { - auto* r = new (std::nothrow) UrmaResource(); - if (!r) { - break; - } - r->jfce = urma_create_jfce(ctx); - if (!r->jfce || - (!FLAGS_urma_use_polling && r->jfce->fd < 0)) { - delete r; - break; - } - urma_jfc_cfg_t jfc_cfg{}; - jfc_cfg.depth = static_cast(FLAGS_urma_sq_size + FLAGS_urma_rq_size); - jfc_cfg.jfce = r->jfce; - r->jfc = urma_create_jfc(ctx, &jfc_cfg); - if (!r->jfc) { - delete r; - break; - } - urma_jfr_cfg_t jfr_cfg{}; - jfr_cfg.depth = static_cast(FLAGS_urma_rq_size); - jfr_cfg.trans_mode = URMA_TM_RM; - jfr_cfg.max_sge = 1; - jfr_cfg.min_rnr_timer = URMA_TYPICAL_MIN_RNR_TIMER; - jfr_cfg.jfc = r->jfc; - r->jfr = urma_create_jfr(ctx, &jfr_cfg); - if (!r->jfr) { - delete r; + auto resource = LocalJettyResources::Create(config); + if (!resource) { break; } - urma_jetty_cfg_t jetty_cfg{}; - jetty_cfg.flag.bs.share_jfr = 1; - jetty_cfg.jfs_cfg.depth = static_cast(FLAGS_urma_sq_size); - jetty_cfg.jfs_cfg.trans_mode = URMA_TM_RM; - jetty_cfg.jfs_cfg.priority = GetUrmaJettyPriority(); - jetty_cfg.jfs_cfg.max_sge = - static_cast(GetUrmaMaxSge()); - jetty_cfg.jfs_cfg.rnr_retry = URMA_TYPICAL_RNR_RETRY; - jetty_cfg.jfs_cfg.err_timeout = URMA_TYPICAL_ERR_TIMEOUT; - jetty_cfg.jfs_cfg.jfc = r->jfc; - jetty_cfg.shared.jfr = r->jfr; - jetty_cfg.shared.jfc = r->jfc; - r->jetty = urma_create_jetty(ctx, &jetty_cfg); - if (!r->jetty) { - delete r; - break; - } - r->next = g_prepared_list; - g_prepared_list = r; - ++g_prepared_cnt; + prepared.push_back(std::move(resource)); } return 0; } void UrmaEndpoint::GlobalRelease() { + PreparedResources prepared; { BAIDU_SCOPED_LOCK(g_prepared_mutex); - while (g_prepared_list) { - UrmaResource* next = g_prepared_list->next; - delete g_prepared_list; - g_prepared_list = next; - } - g_prepared_cnt = 0; + prepared.swap(PreparedJettyResources()); } + prepared.clear(); for (size_t tag = 0; tag < _poller_groups.size(); ++tag) { PollingModeRelease(static_cast(tag)); } diff --git a/src/brpc/urma/urma_endpoint.h b/src/brpc/urma/urma_endpoint.h index e5aed5ac66..3d4a36fe4e 100644 --- a/src/brpc/urma/urma_endpoint.h +++ b/src/brpc/urma/urma_endpoint.h @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -38,6 +39,7 @@ #include "urma_types.h" #include "brpc/urma/urma_handshake.h" #include "brpc/urma/urma_handshake.pb.h" +#include "brpc/urma/urma_resource.h" namespace brpc { @@ -75,27 +77,6 @@ class UrmaConnect : public AppConnect { int _error{0}; }; -// POD holder for the URMA kernel objects backing one connection: -// - the send jetty (JFS), shared-JFR jetty, JFR, JFC, and (event mode) JFCE -// - the imported peer jetty and peer segment -// Mirrors the role of rdma::RdmaResource, but the URMA object graph is a -// little different (no separate QP/CQ split). -struct UrmaResource { - UrmaResource* next{nullptr}; // singly-linked list for the prepared pool - - urma_jfc_t* jfc{nullptr}; - urma_jfce_t* jfce{nullptr}; // event mode only; null in polling mode - urma_jfr_t* jfr{nullptr}; - urma_jetty_t* jetty{nullptr}; - // Imported peer objects (created per-connection, not pooled). - urma_target_jetty_t* remote_jetty{nullptr}; - urma_target_seg_t* remote_seg{nullptr}; - - UrmaResource() = default; - ~UrmaResource(); - DISALLOW_COPY_AND_ASSIGN(UrmaResource); -}; - // One per Socket. Carries the handshake state machine, the send/recv // windows, and the registered buffer bookkeeping. Cache-line padded to avoid // false sharing between connections. @@ -278,8 +259,9 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { butil::atomic _pending_received_bytes{0}; butil::Mutex _dispatch_mutex; - // The URMA resources (jetty / jfc / jfr / jfce / imported peer objects). - UrmaResource* _resource{nullptr}; + // Local queues and the remote handles imported for this connection. + std::unique_ptr _resource; + PeerResources _peer; // The SocketId wrapping the JFCE fd (event mode) or a synthetic carrier // (polling mode). PollCq is the edge-triggered callback on this socket. diff --git a/src/brpc/urma/urma_resource.cpp b/src/brpc/urma/urma_resource.cpp new file mode 100644 index 0000000000..85c3240f8c --- /dev/null +++ b/src/brpc/urma/urma_resource.cpp @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "brpc/urma/urma_resource.h" + +#if BRPC_WITH_URMA + +#include +#include + +#include "urma_api.h" +#include "butil/logging.h" + +namespace brpc { +namespace urma { + +bool LocalJettyConfig::operator==(const LocalJettyConfig& other) const { + return context == other.context && sq_depth == other.sq_depth && + rq_depth == other.rq_depth && max_sge == other.max_sge && + priority == other.priority && use_polling == other.use_polling; +} + +std::unique_ptr LocalJettyResources::Create( + const LocalJettyConfig& config) { + std::unique_ptr resource( + new (std::nothrow) LocalJettyResources(config)); + if (!resource) { + errno = ENOMEM; + return nullptr; + } + auto failed = [&resource](const char* operation) + -> std::unique_ptr { + const int error = errno ? errno : EIO; + PLOG(ERROR) << operation; + resource.reset(); + errno = error; + return nullptr; + }; + + // The SDK requires every JFC to reference a JFCE. Polling mode does + // not arm or consume it, but still supplies the required object. + resource->jfce = urma_create_jfce(config.context); + if (!resource->jfce) { + return failed("urma_create_jfce"); + } + if (!config.use_polling && resource->jfce->fd < 0) { + errno = ENODEV; + return failed("Invalid URMA JFCE fd"); + } + + urma_jfc_cfg_t jfc_cfg{}; + jfc_cfg.depth = static_cast(config.sq_depth + config.rq_depth); + jfc_cfg.jfce = resource->jfce; + resource->jfc = urma_create_jfc(config.context, &jfc_cfg); + if (!resource->jfc) { + return failed("urma_create_jfc"); + } + + urma_jfr_cfg_t jfr_cfg{}; + jfr_cfg.depth = static_cast(config.rq_depth); + jfr_cfg.trans_mode = URMA_TM_RM; + jfr_cfg.max_sge = 1; + jfr_cfg.min_rnr_timer = URMA_TYPICAL_MIN_RNR_TIMER; + jfr_cfg.jfc = resource->jfc; + resource->jfr = urma_create_jfr(config.context, &jfr_cfg); + if (!resource->jfr) { + return failed("urma_create_jfr"); + } + + urma_jetty_cfg_t jetty_cfg{}; + jetty_cfg.flag.bs.share_jfr = 1; + jetty_cfg.jfs_cfg.depth = static_cast(config.sq_depth); + jetty_cfg.jfs_cfg.trans_mode = URMA_TM_RM; + jetty_cfg.jfs_cfg.priority = config.priority; + jetty_cfg.jfs_cfg.max_sge = config.max_sge; + jetty_cfg.jfs_cfg.rnr_retry = URMA_TYPICAL_RNR_RETRY; + jetty_cfg.jfs_cfg.err_timeout = URMA_TYPICAL_ERR_TIMEOUT; + jetty_cfg.jfs_cfg.jfc = resource->jfc; + jetty_cfg.shared.jfr = resource->jfr; + jetty_cfg.shared.jfc = resource->jfc; + resource->jetty = urma_create_jetty(config.context, &jetty_cfg); + if (!resource->jetty) { + return failed("urma_create_jetty"); + } + return resource; +} + +LocalJettyResources::~LocalJettyResources() { + if (jetty) { + urma_delete_jetty(jetty); + } + if (jfr) { + urma_delete_jfr(jfr); + } + if (jfc) { + urma_delete_jfc(jfc); + } + if (jfce) { + urma_delete_jfce(jfce); + } +} + +PeerResources::~PeerResources() { + Reset(); +} + +void PeerResources::Reset() { + if (jetty) { + urma_unimport_jetty(jetty); + jetty = nullptr; + } + if (segment) { + urma_unimport_seg(segment); + segment = nullptr; + } +} + +} // namespace urma +} // namespace brpc + +#endif // BRPC_WITH_URMA diff --git a/src/brpc/urma/urma_resource.h b/src/brpc/urma/urma_resource.h new file mode 100644 index 0000000000..8dabf473be --- /dev/null +++ b/src/brpc/urma/urma_resource.h @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_URMA_RESOURCE_H +#define BRPC_URMA_RESOURCE_H + +#if BRPC_WITH_URMA + +#include +#include + +#include "urma_types.h" +#include "butil/macros.h" + +namespace brpc { +namespace urma { + +struct LocalJettyConfig { + urma_context_t* context; + uint16_t sq_depth; + uint16_t rq_depth; + uint8_t max_sge; + uint8_t priority; + bool use_polling; + + bool operator==(const LocalJettyConfig& other) const; +}; + +// Owns the local kernel objects shared by the cold and prepared paths. +struct LocalJettyResources { + static std::unique_ptr Create( + const LocalJettyConfig& config); + ~LocalJettyResources(); + + const LocalJettyConfig config; + urma_jfce_t* jfce{nullptr}; + urma_jfc_t* jfc{nullptr}; + urma_jfr_t* jfr{nullptr}; + urma_jetty_t* jetty{nullptr}; + +private: + explicit LocalJettyResources(const LocalJettyConfig& cfg) : config(cfg) {} + DISALLOW_COPY_AND_ASSIGN(LocalJettyResources); +}; + +// Remote imports belong to a single handshake and never enter the prepared pool. +struct PeerResources { + PeerResources() = default; + ~PeerResources(); + void Reset(); + + urma_target_jetty_t* jetty{nullptr}; + urma_target_seg_t* segment{nullptr}; + +private: + DISALLOW_COPY_AND_ASSIGN(PeerResources); +}; + +} // namespace urma +} // namespace brpc + +#endif // BRPC_WITH_URMA +#endif // BRPC_URMA_RESOURCE_H diff --git a/test/brpc_urma_unittest.cpp b/test/brpc_urma_unittest.cpp index d27367a504..dbfb8dc8c4 100644 --- a/test/brpc_urma_unittest.cpp +++ b/test/brpc_urma_unittest.cpp @@ -21,12 +21,17 @@ #include #if BRPC_WITH_URMA +#include +#include + #include "butil/atomicops.h" +#include "butil/memory/scope_guard.h" #include "butil/sys_byteorder.h" #include "urma_api.h" #include "brpc/urma/urma_handshake.h" #include "brpc/urma/urma_handshake.pb.h" #include "brpc/urma/urma_helper.h" +#include "brpc/urma/urma_resource.h" #include "urma_types.h" using namespace brpc; @@ -352,6 +357,45 @@ TEST_F(UrmaMockTest, create_context_and_query_device) { urma_free_device_list(devices); } +TEST_F(UrmaMockTest, creates_local_queues_with_requested_config) { + int num_devices = 0; + urma_device_t** devices = urma_get_device_list(&num_devices); + ASSERT_NE(nullptr, devices); + auto free_devices = butil::MakeScopeGuard( + [&] { urma_free_device_list(devices); }); + ASSERT_GT(num_devices, 0); + urma_context_t* ctx = urma_create_context(devices[0], 0); + ASSERT_NE(nullptr, ctx); + auto delete_context = butil::MakeScopeGuard( + [&] { EXPECT_EQ(URMA_SUCCESS, urma_delete_context(ctx)); }); + + for (bool polling : {false, true}) { + const urma::LocalJettyConfig config{ctx, 32, 64, 2, 1, polling}; + auto queues = urma::LocalJettyResources::Create(config); + ASSERT_NE(nullptr, queues); + EXPECT_TRUE(config == queues->config); + EXPECT_EQ(96u, queues->jfc->jfc_cfg.depth); + EXPECT_EQ(32u, queues->jetty->jetty_cfg.jfs_cfg.depth); + EXPECT_EQ(2u, queues->jetty->jetty_cfg.jfs_cfg.max_sge); + EXPECT_EQ(1u, queues->jetty->jetty_cfg.jfs_cfg.priority); + EXPECT_EQ(queues->jfr, queues->jetty->jetty_cfg.shared.jfr); + EXPECT_EQ(queues->jfc, queues->jetty->jetty_cfg.shared.jfc); + + auto different_depth = config; + ++different_depth.rq_depth; + EXPECT_FALSE(different_depth == queues->config); + auto different_mode = config; + different_mode.use_polling = !polling; + EXPECT_FALSE(different_mode == queues->config); + + const int fd = queues->jfce->fd; + ASSERT_GE(fcntl(fd, F_GETFD), 0); + queues.reset(); + EXPECT_EQ(-1, fcntl(fd, F_GETFD)); + EXPECT_EQ(EBADF, errno); + } +} + TEST_F(UrmaMockTest, rejects_send_without_target_jetty) { // A SEND without a target jetty is invalid. Keeping the mock strict here // prevents tests from relying on input that a real provider cannot post.