Skip to content

xio_messenger: Enable flow-control on total msgs and bytes per connectio... #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
vuhuong wants to merge 1 commit into from
Closed

xio_messenger: Enable flow-control on total msgs and bytes per connectio... #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/msg/xio/XioConnection.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,15 +416,15 @@ int XioConnection::on_ow_msg_send_complete(struct xio_session *session,

void XioConnection::msg_send_fail(XioMsg *xmsg, int code)
{
ldout(msgr->cct,4) << "xio_send_msg FAILED " << &xmsg->req_0.msg << " code=" << code <<
ldout(msgr->cct,0) << "xio_send_msg FAILED " << &xmsg->req_0.msg << " code=" << code <<
" (" << xio_strerror(code) << ")" << dendl;
/* return refs taken for each xio_msg */
xmsg->put_msg_refs();
} /* msg_send_fail */

void XioConnection::msg_release_fail(struct xio_msg *msg, int code)
{
ldout(msgr->cct,4) << "xio_release_msg FAILED " << msg << "code=" << code <<
ldout(msgr->cct,0) << "xio_release_msg FAILED " << msg << "code=" << code <<
" (" << xio_strerror(code) << ")" << dendl;
} /* msg_release_fail */

Expand Down
13 changes: 13 additions & 0 deletions src/msg/xio/XioMessenger.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,12 +289,25 @@ XioMessenger::XioMessenger(CephContext *cct, entity_name_t name,
xio_set_opt(NULL, XIO_OPTLEVEL_ACCELIO, XIO_OPTNAME_MAX_OUT_IOVLEN,
&xopt, sizeof(xopt));

/* enable flow-control */
xopt = 1;
xio_set_opt(NULL, XIO_OPTLEVEL_ACCELIO, XIO_OPTNAME_ENABLE_FLOW_CONTROL,
&xopt, sizeof(xopt));

xopt = cct->_conf->xio_queue_depth; // defaults to 512
xio_set_opt(NULL, XIO_OPTLEVEL_ACCELIO, XIO_OPTNAME_SND_QUEUE_DEPTH_MSGS,
&xopt, sizeof(xopt));
xio_set_opt(NULL, XIO_OPTLEVEL_ACCELIO, XIO_OPTNAME_RCV_QUEUE_DEPTH_MSGS,
&xopt, sizeof(xopt));

/* flow-control total bytes per connection; should get from messenger's policy */
uint64_t bytes_opt;
bytes_opt = (2 << 27); // 256MB
xio_set_opt(NULL, XIO_OPTLEVEL_ACCELIO, XIO_OPTNAME_SND_QUEUE_DEPTH_BYTES,
&bytes_opt, sizeof(bytes_opt));
xio_set_opt(NULL, XIO_OPTLEVEL_ACCELIO, XIO_OPTNAME_RCV_QUEUE_DEPTH_BYTES,
&bytes_opt, sizeof(bytes_opt));

/* and set threshold for buffer callouts */
xopt = 16384;
xio_set_opt(NULL, XIO_OPTLEVEL_ACCELIO, XIO_OPTNAME_MAX_INLINE_DATA,
Expand Down
3 changes: 3 additions & 0 deletions src/msg/xio/XioPortal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,9 @@ class XioPortal : public Thread
}
}
if (unlikely(code)) {
std::cout << "xio_send_msg FAILED " << &xmsg->req_0.msg <<
" code=" << code <<
" (" << xio_strerror(code) << ")" << std::endl;
xs->xcon->msg_send_fail(xmsg, code);
} else {
xs->xcon->send.set(msg->timestamp); // need atomic?
Expand Down