From 1f7b4e670bc70e5583958e6c25e473fbd683228d Mon Sep 17 00:00:00 2001 From: Oknet Date: Fri, 27 May 2016 18:26:19 +0800 Subject: [PATCH 1/2] TS-4487: fix write_to_net_io issues --- iocore/net/UnixNetVConnection.cc | 78 +++++++++++++++++--------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index 2a903cf9968..f48aba69ce3 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -545,57 +545,61 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) vc->write.triggered = 0; write_signal_error(nh, vc, (int)-r); return; - } else { - int wbe_event = vc->write_buffer_empty_event; // save so we can clear if needed. + } - NET_SUM_DYN_STAT(net_write_bytes_stat, r); + int wbe_event = vc->write_buffer_empty_event; // save so we can clear if needed. - // Remove data from the buffer and signal continuation. - ink_assert(buf.reader()->read_avail() >= r); - buf.reader()->consume(r); - ink_assert(buf.reader()->read_avail() >= 0); - s->vio.ndone += r; + NET_SUM_DYN_STAT(net_write_bytes_stat, r); - // If the empty write buffer trap is set, clear it. - if (!(buf.reader()->is_read_avail_more_than(0))) - vc->write_buffer_empty_event = 0; + // Remove data from the buffer and signal continuation. + ink_assert(buf.reader()->read_avail() >= r); + buf.reader()->consume(r); + ink_assert(buf.reader()->read_avail() >= 0); + s->vio.ndone += r; - net_activity(vc, thread); - // If there are no more bytes to write, signal write complete, - ink_assert(ntodo >= 0); - if (s->vio.ntodo() <= 0) { - write_signal_done(VC_EVENT_WRITE_COMPLETE, nh, vc); - return; - } else if (signalled && (wbe_event != vc->write_buffer_empty_event)) { + // If the empty write buffer trap is set, clear it. + if (!(buf.reader()->is_read_avail_more_than(0))) + vc->write_buffer_empty_event = 0; + + net_activity(vc, thread); + // If there are no more bytes to write, signal write complete, + ink_assert(ntodo >= 0); + int e = 0; + if (s->vio.ntodo() <= 0) { + write_signal_done(VC_EVENT_WRITE_COMPLETE, nh, vc); + return; + } else { + if (!signalled) { + e = VC_EVENT_WRITE_READY; + } else if (wbe_event != vc->write_buffer_empty_event) { // @a signalled means we won't send an event, and the event values differing means we // had a write buffer trap and cleared it, so we need to send it now. - if (write_signal_and_update(wbe_event, vc) != EVENT_CONT) - return; - } else if (!signalled) { - if (write_signal_and_update(VC_EVENT_WRITE_READY, vc) != EVENT_CONT) { - return; - } - - // change of lock... don't look at shared variables! - if (lock.get_mutex() != s->vio.mutex.get()) { - write_reschedule(nh, vc); - return; - } + e = wbe_event; } - - if (!buf.reader()->read_avail()) { - write_disable(nh, vc); + } + if (e) { + if (write_signal_and_update(e, vc) != EVENT_CONT) { return; } - if ((needs & EVENTIO_WRITE) == EVENTIO_WRITE) { + // change of lock... don't look at shared variables! + if (lock.get_mutex() != s->vio.mutex.get()) { write_reschedule(nh, vc); + return; } - if ((needs & EVENTIO_READ) == EVENTIO_READ) { - read_reschedule(nh, vc); - } + } + + if (needs==0 && !buf.reader()->read_avail()) { + write_disable(nh, vc); return; } + + if ((needs & EVENTIO_WRITE) == EVENTIO_WRITE) { + write_reschedule(nh, vc); + } + if ((needs & EVENTIO_READ) == EVENTIO_READ) { + read_reschedule(nh, vc); + } } bool From 97c4eebbc515c195c0f78232ebd29b0623a2a7ed Mon Sep 17 00:00:00 2001 From: Oknet Date: Wed, 11 May 2016 23:29:02 +0000 Subject: [PATCH 2/2] Update to #629 based patch --- iocore/net/P_SSLNetVConnection.h | 3 +- iocore/net/P_UnixNetVConnection.h | 3 +- iocore/net/SSLNetVConnection.cc | 261 +++++++++++++----------------- iocore/net/UnixNetVConnection.cc | 72 ++++----- 4 files changed, 147 insertions(+), 192 deletions(-) diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h index a4ae8b3c7e5..005e581c273 100644 --- a/iocore/net/P_SSLNetVConnection.h +++ b/iocore/net/P_SSLNetVConnection.h @@ -120,8 +120,7 @@ class SSLNetVConnection : public UnixNetVConnection int sslServerHandShakeEvent(int &err); int sslClientHandShakeEvent(int &err); virtual void net_read_io(NetHandler *nh, EThread *lthread); - virtual int64_t load_buffer_and_write(int64_t towrite, int64_t &wattempted, int64_t &total_written, MIOBufferAccessor &buf, - int &needs); + virtual int64_t load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs); void registerNextProtocolSet(const SSLNextProtocolSet *); virtual void do_io_close(int lerrno = -1); diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 14786eb6f54..d09ba494e86 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -208,8 +208,7 @@ class UnixNetVConnection : public NetVConnection (void)state; } virtual void net_read_io(NetHandler *nh, EThread *lthread); - virtual int64_t load_buffer_and_write(int64_t towrite, int64_t &wattempted, int64_t &total_written, MIOBufferAccessor &buf, - int &needs); + virtual int64_t load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs); void readDisable(NetHandler *nh); void readSignalError(NetHandler *nh, int err); int readSignalDone(int event, NetHandler *nh); diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index 27dc985d879..c6e0e5ac760 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -193,131 +193,121 @@ ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret) { NetState *s = &sslvc->read; MIOBufferAccessor &buf = s->vio.buffer; - IOBufferBlock *b = buf.writer()->first_write_block(); int event = SSL_READ_ERROR_NONE; int64_t bytes_read = 0; - int64_t block_write_avail = 0; ssl_error_t sslErr = SSL_ERROR_NONE; int64_t nread = 0; bool trace = sslvc->getSSLTrace(); Debug("ssl", "trace=%s", trace ? "TRUE" : "FALSE"); - for (bytes_read = 0; (b != 0) && (sslErr == SSL_ERROR_NONE); b = b->next.get()) { - block_write_avail = b->write_avail(); + bytes_read = 0; + while (sslErr == SSL_ERROR_NONE) { + int64_t block_write_avail = buf.writer()->block_write_avail(); + if (block_write_avail <= 0) { + buf.writer()->add_block(); + block_write_avail = buf.writer()->block_write_avail(); + if (block_write_avail <= 0) { + Warning("Cannot add new block"); + break; + } + } Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] b->write_avail()=%" PRId64, block_write_avail); + char *current_block = buf.writer()->end(); + sslErr = SSLReadBuffer(sslvc->ssl, current_block, block_write_avail, nread); - int64_t offset = 0; - // while can be replaced with if - need to test what works faster with openssl - while (block_write_avail > 0) { - sslErr = SSLReadBuffer(sslvc->ssl, b->end() + offset, block_write_avail, nread); - - Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] nread=%d", (int)nread); - if (!sslvc->origin_trace) { - TraceIn((0 < nread && trace), sslvc->get_remote_addr(), sslvc->get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", (int)nread, - (int)nread, b->end() + offset); - } else { - char origin_trace_ip[INET6_ADDRSTRLEN]; - ats_ip_ntop(sslvc->origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - TraceIn((0 < nread && trace), sslvc->get_remote_addr(), sslvc->get_remote_port(), "CLIENT %s:%d\ttbytes=%d\n%.*s", - origin_trace_ip, sslvc->origin_trace_port, (int)nread, (int)nread, b->end() + offset); - } + Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] nread=%d", (int)nread); + if (!sslvc->origin_trace) { + TraceIn((0 < nread && trace), sslvc->get_remote_addr(), sslvc->get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", (int)nread, + (int)nread, current_block); + } else { + char origin_trace_ip[INET6_ADDRSTRLEN]; + ats_ip_ntop(sslvc->origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); + TraceIn((0 < nread && trace), sslvc->get_remote_addr(), sslvc->get_remote_port(), "CLIENT %s:%d\ttbytes=%d\n%.*s", + origin_trace_ip, sslvc->origin_trace_port, (int)nread, (int)nread, current_block); + } - switch (sslErr) { - case SSL_ERROR_NONE: + switch (sslErr) { + case SSL_ERROR_NONE: #if DEBUG - SSLDebugBufferPrint("ssl_buff", b->end() + offset, nread, "SSL Read"); + SSLDebugBufferPrint("ssl_buff", current_block, nread, "SSL Read"); #endif - - ink_assert(nread); - - bytes_read += nread; - offset += nread; - block_write_avail -= nread; - ink_assert(block_write_avail >= 0); - - continue; - - case SSL_ERROR_WANT_WRITE: - event = SSL_WRITE_WOULD_BLOCK; - SSL_INCREMENT_DYN_STAT(ssl_error_want_write); - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(write)"); - break; - case SSL_ERROR_WANT_READ: - event = SSL_READ_WOULD_BLOCK; - SSL_INCREMENT_DYN_STAT(ssl_error_want_read); - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read)"); - break; - case SSL_ERROR_WANT_X509_LOOKUP: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Want X509 lookup"); - event = SSL_READ_WOULD_BLOCK; - SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read/x509 lookup)"); - break; - case SSL_ERROR_SYSCALL: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Syscall Error: %s", strerror(errno)); - SSL_INCREMENT_DYN_STAT(ssl_error_syscall); - if (nread != 0) { - // not EOF - event = SSL_READ_ERROR; - ret = errno; - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, underlying IO error: %s", strerror(errno)); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Underlying IO error: %d", errno); - } else { - // then EOF observed, treat it as EOS - event = SSL_READ_EOS; - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, EOF observed violating SSL protocol"); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "EOF observed violating SSL protocol"); - } - break; - case SSL_ERROR_ZERO_RETURN: - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Connection closed by peer"); - event = SSL_READ_EOS; - SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_ZERO_RETURN"); - break; - case SSL_ERROR_SSL: - default: { - Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SSL and default case"); - char buf[512]; - unsigned long e = ERR_peek_last_error(); - ERR_error_string_n(e, buf, sizeof(buf)); - TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", - sslErr, e, buf, errno); + ink_assert(nread); + bytes_read += nread; + if (nread > 0) { + buf.writer()->fill(nread); // Tell the buffer, we've used the bytes + } + break; + case SSL_ERROR_WANT_WRITE: + event = SSL_WRITE_WOULD_BLOCK; + SSL_INCREMENT_DYN_STAT(ssl_error_want_write); + Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(write)"); + break; + case SSL_ERROR_WANT_READ: + event = SSL_READ_WOULD_BLOCK; + SSL_INCREMENT_DYN_STAT(ssl_error_want_read); + Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read)"); + break; + case SSL_ERROR_WANT_X509_LOOKUP: + TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Want X509 lookup"); + event = SSL_READ_WOULD_BLOCK; + SSL_INCREMENT_DYN_STAT(ssl_error_want_x509_lookup); + Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_WOULD_BLOCK(read/x509 lookup)"); + break; + case SSL_ERROR_SYSCALL: + TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Syscall Error: %s", strerror(errno)); + SSL_INCREMENT_DYN_STAT(ssl_error_syscall); + if (nread != 0) { + // not EOF event = SSL_READ_ERROR; ret = errno; - SSL_CLR_ERR_INCR_DYN_STAT(sslvc, ssl_error_ssl, "[SSL_NetVConnection::ssl_read_from_net]: errno=%d", errno); - } break; - } // switch + Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, underlying IO error: %s", strerror(errno)); + TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Underlying IO error: %d", errno); + } else { + // then EOF observed, treat it as EOS + // Error("[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, EOF observed violating SSL protocol"); + TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "EOF observed violating SSL protocol"); + } + break; + case SSL_ERROR_ZERO_RETURN: + TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "Connection closed by peer"); + event = SSL_READ_EOS; + SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); + Debug("ssl.error", "[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_ZERO_RETURN"); break; - } // while( block_write_avail > 0 ) - } // for ( bytes_read = 0; (b != 0); b = b->next) + case SSL_ERROR_SSL: + default: { + char buf[512]; + unsigned long e = ERR_peek_last_error(); + ERR_error_string_n(e, buf, sizeof(buf)); + TraceIn(trace, sslvc->get_remote_addr(), sslvc->get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", + sslErr, e, buf, errno); + event = SSL_READ_ERROR; + ret = errno; + SSL_CLR_ERR_INCR_DYN_STAT(sslvc, ssl_error_ssl, "[SSL_NetVConnection::ssl_read_from_net]: errno=%d", errno); + } break; + } // switch + } // while if (bytes_read > 0) { Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] bytes_read=%" PRId64, bytes_read); - buf.writer()->fill(bytes_read); s->vio.ndone += bytes_read; sslvc->netActivity(lthread); ret = bytes_read; - if (s->vio.ntodo() <= 0) { - event = SSL_READ_COMPLETE; - } else { - event = SSL_READ_READY; - } - } else // if( bytes_read > 0 ) - { + event = (s->vio.ntodo() <= 0) ? SSL_READ_COMPLETE : SSL_READ_READY; + } else { // if( bytes_read > 0 ) #if defined(_DEBUG) if (bytes_read == 0) { Debug("ssl", "[SSL_NetVConnection::ssl_read_from_net] bytes_read == 0"); } #endif } - return (event); + return event; } /** @@ -647,17 +637,14 @@ SSLNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) } int64_t -SSLNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, int64_t &total_written, MIOBufferAccessor &buf, - int &needs) +SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs) { - int64_t r = 0; + int64_t try_to_write; + int64_t num_really_written = 0; int64_t l = 0; uint32_t dynamic_tls_record_size = 0; ssl_error_t err = SSL_ERROR_NONE; - - // XXX Rather than dealing with the block directly, we should use the IOBufferReader API. - int64_t offset = buf.reader()->start_offset; - IOBufferBlock *b = buf.reader()->block.get(); + char *current_block = buf.reader()->start(); // Dynamic TLS record sizing ink_hrtime now = 0; @@ -674,21 +661,17 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, i } if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == this->attributes) { - return this->super::load_buffer_and_write(towrite, wattempted, total_written, buf, needs); + return this->super::load_buffer_and_write(towrite, buf, total_written, needs); } bool trace = getSSLTrace(); Debug("ssl", "trace=%s", trace ? "TRUE" : "FALSE"); do { - // check if we have done this block - l = b->read_avail(); - l -= offset; - if (l <= 0) { - offset = -l; - b = b->next.get(); - continue; - } + // What is remaining left in the next block? + l = buf.reader()->block_read_avail(); + current_block = buf.reader()->start(); + // check if to amount to write exceeds that in this buffer int64_t wavail = towrite - total_written; @@ -699,7 +682,6 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, i // TS-2365: If the SSL max record size is set and we have // more data than that, break this into smaller write // operations. - int64_t orig_l = l; if (SSLConfigParams::ssl_maxrecord > 0 && l > SSLConfigParams::ssl_maxrecord) { l = SSLConfigParams::ssl_maxrecord; } else if (SSLConfigParams::ssl_maxrecord == -1) { @@ -719,50 +701,39 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, i break; } - wattempted = l; - total_written += l; + try_to_write = l; + num_really_written = 0; Debug("ssl", "SSLNetVConnection::loadBufferAndCallWrite, before SSLWriteBuffer, l=%" PRId64 ", towrite=%" PRId64 ", b=%p", l, - towrite, b); - err = SSLWriteBuffer(ssl, b->start() + offset, l, r); + towrite, current_block); + err = SSLWriteBuffer(ssl, current_block, l, num_really_written); if (!origin_trace) { - TraceOut((0 < r && trace), get_remote_addr(), get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", (int)r, (int)r, - b->start() + offset); + TraceOut((0 < num_really_written && trace), get_remote_addr(), get_remote_port(), "WIRE TRACE\tbytes=%d\n%.*s", + (int)num_really_written, (int)num_really_written, current_block); } else { char origin_trace_ip[INET6_ADDRSTRLEN]; ats_ip_ntop(origin_trace_addr, origin_trace_ip, sizeof(origin_trace_ip)); - TraceOut((0 < r && trace), get_remote_addr(), get_remote_port(), "CLIENT %s:%d\ttbytes=%d\n%.*s", origin_trace_ip, - origin_trace_port, (int)r, (int)r, b->start() + offset); + TraceOut((0 < num_really_written && trace), get_remote_addr(), get_remote_port(), "CLIENT %s:%d\ttbytes=%d\n%.*s", + origin_trace_ip, origin_trace_port, (int)num_really_written, (int)num_really_written, current_block); } - if (r == l) { - wattempted = total_written; - } - if (l == orig_l) { - // on to the next block - offset = 0; - b = b->next.get(); - } else { - offset += l; + // We wrote all that we thought we should + if (num_really_written > 0) { + total_written += num_really_written; + buf.reader()->consume(num_really_written); } - Debug("ssl", "SSLNetVConnection::loadBufferAndCallWrite,Number of bytes written=%" PRId64 " , total=%" PRId64 "", r, - total_written); + Debug("ssl", "SSLNetVConnection::loadBufferAndCallWrite,Number of bytes written=%" PRId64 " , total=%" PRId64 "", + num_really_written, total_written); NET_INCREMENT_DYN_STAT(net_calls_to_write_stat); - } while (r == l && total_written < towrite && b); + } while (num_really_written == try_to_write && total_written < towrite); - if (r > 0) { + if (total_written > 0) { sslLastWriteTime = now; sslTotalBytesSent += total_written; - if (total_written != wattempted) { - Debug("ssl", "SSLNetVConnection::loadBufferAndCallWrite, wrote some bytes, but not all requested."); - // I'm not sure how this could happen. We should have tried and hit an EAGAIN. - needs |= EVENTIO_WRITE; - return (r); - } else { - Debug("ssl", "SSLNetVConnection::loadBufferAndCallWrite, write successful."); - return (total_written); - } + } + if (num_really_written > 0) { + needs |= EVENTIO_WRITE; } else { switch (err) { case SSL_ERROR_NONE: @@ -770,7 +741,7 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, i break; case SSL_ERROR_WANT_READ: needs |= EVENTIO_READ; - r = -EAGAIN; + num_really_written = -EAGAIN; SSL_INCREMENT_DYN_STAT(ssl_error_want_read); Debug("ssl.error", "SSL_write-SSL_ERROR_WANT_READ"); break; @@ -784,20 +755,20 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, i } needs |= EVENTIO_WRITE; - r = -EAGAIN; + num_really_written = -EAGAIN; Debug("ssl.error", "SSL_write-SSL_ERROR_WANT_WRITE"); break; } case SSL_ERROR_SYSCALL: TraceOut(trace, get_remote_addr(), get_remote_port(), "Syscall Error: %s", strerror(errno)); - r = -errno; + num_really_written = -errno; SSL_INCREMENT_DYN_STAT(ssl_error_syscall); Debug("ssl.error", "SSL_write-SSL_ERROR_SYSCALL"); break; // end of stream case SSL_ERROR_ZERO_RETURN: TraceOut(trace, get_remote_addr(), get_remote_port(), "SSL Error: zero return"); - r = -errno; + num_really_written = -errno; SSL_INCREMENT_DYN_STAT(ssl_error_zero_return); Debug("ssl.error", "SSL_write-SSL_ERROR_ZERO_RETURN"); break; @@ -808,12 +779,12 @@ SSLNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, i ERR_error_string_n(e, buf, sizeof(buf)); TraceIn(trace, get_remote_addr(), get_remote_port(), "SSL Error: sslErr=%d, ERR_get_error=%ld (%s) errno=%d", err, e, buf, errno); - r = -errno; + num_really_written = -errno; SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSL_write-SSL_ERROR_SSL errno=%d", errno); } break; } - return (r); } + return num_really_written; } SSLNetVConnection::SSLNetVConnection() diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index f48aba69ce3..59d2d8a9781 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -510,17 +510,15 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) return; } - int64_t total_written = 0, wattempted = 0; int needs = 0; - int64_t r = vc->load_buffer_and_write(towrite, wattempted, total_written, buf, needs); + int64_t total_written = 0; + int64_t r = vc->load_buffer_and_write(towrite, buf, total_written, needs); - // if we have already moved some bytes successfully, summarize in r - if (total_written != wattempted) { - if (r <= 0) - r = total_written - wattempted; - else - r = total_written - wattempted + r; + if (total_written > 0) { + NET_SUM_DYN_STAT(net_write_bytes_stat, total_written); + s->vio.ndone += total_written; } + // check for errors if (r <= 0) { // if the socket was not ready,add to WaitList if (r == -EAGAIN || r == -ENOTCONN) { @@ -543,20 +541,12 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) return; } vc->write.triggered = 0; - write_signal_error(nh, vc, (int)-r); + write_signal_error(nh, vc, (int)-total_written); return; } - + // Wrote data. Finished without error int wbe_event = vc->write_buffer_empty_event; // save so we can clear if needed. - NET_SUM_DYN_STAT(net_write_bytes_stat, r); - - // Remove data from the buffer and signal continuation. - ink_assert(buf.reader()->read_avail() >= r); - buf.reader()->consume(r); - ink_assert(buf.reader()->read_avail() >= 0); - s->vio.ndone += r; - // If the empty write buffer trap is set, clear it. if (!(buf.reader()->is_read_avail_more_than(0))) vc->write_buffer_empty_event = 0; @@ -589,7 +579,7 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) } } - if (needs==0 && !buf.reader()->read_avail()) { + if (needs==0 && !(buf.reader()->is_read_avail_more_than(0))) { write_disable(nh, vc); return; } @@ -950,28 +940,22 @@ UnixNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) // (SSL read does not support overlapped i/o) // without duplicating all the code in write_to_net. int64_t -UnixNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, int64_t &total_written, MIOBufferAccessor &buf, - int &needs) +UnixNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs) { int64_t r = 0; - - // XXX Rather than dealing with the block directly, we should use the IOBufferReader API. - int64_t offset = buf.reader()->start_offset; - IOBufferBlock *b = buf.reader()->block.get(); + int64_t try_to_write = 0; + IOBufferReader *tmp_reader = buf.reader()->clone(); do { IOVec tiovec[NET_MAX_IOV]; unsigned niov = 0; - int64_t total_written_last = total_written; - while (b && niov < NET_MAX_IOV) { + try_to_write = 0; + while (niov < NET_MAX_IOV) { // check if we have done this block - int64_t l = b->read_avail(); - l -= offset; - if (l <= 0) { - offset = -l; - b = b->next.get(); - continue; - } + int64_t l = tmp_reader->block_read_avail(); + if (l <= 0) + break; + char *current_block = tmp_reader->start(); // check if to amount to write exceeds that in this buffer int64_t wavail = towrite - total_written; @@ -983,18 +967,14 @@ UnixNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, break; } - total_written += l; // build an iov entry tiovec[niov].iov_len = l; - tiovec[niov].iov_base = b->start() + offset; + try_to_write += l; + tiovec[niov].iov_base = current_block; niov++; - // on to the next block - offset = 0; - b = b->next.get(); + tmp_reader->consume(l); } - wattempted = total_written - total_written_last; - ink_assert(niov > 0); ink_assert(niov <= countof(tiovec)); r = socketManager.writev(con.fd, &tiovec[0], niov); @@ -1015,14 +995,20 @@ UnixNetVConnection::load_buffer_and_write(int64_t towrite, int64_t &wattempted, strerror(errno)); } } + if (r > 0) { + buf.reader()->consume(r); + } + total_written += r; ProxyMutex *mutex = thread->mutex.get(); NET_INCREMENT_DYN_STAT(net_calls_to_write_stat); - } while (r == wattempted && total_written < towrite); + } while (r == try_to_write && total_written < towrite); + + tmp_reader->dealloc(); needs |= EVENTIO_WRITE; - return (r); + return r; } void