Skip to content

N
nghttp2
Commit 4eced8a3 authored by Tatsuhiro Tsujikawa's avatar Tatsuhiro Tsujikawa
Browse files
Options

Build without HTTP/3 support

parent f46984d2
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 365 additions and 107 deletions
CMakeLists.txt
View file @ 4eced8a3
......@@ -448,6 +448,10 @@ foreach(name
configure_file("${name}.in" "${name}" @ONLY)
endforeach()
if(APPLE)
add_definitions(-D__APPLE_USE_RFC_3542)
endif()
include_directories(
"${CMAKE_CURRENT_BINARY_DIR}" # for config.h
)
......
configure.ac
View file @ 4eced8a3
......@@ -107,6 +107,11 @@ AC_ARG_ENABLE([lib-only],
[Build libnghttp2 only. This is a short hand for --disable-app --disable-examples --disable-hpack-tools --disable-python-bindings])],
[request_lib_only=$enableval], [request_lib_only=no])
AC_ARG_ENABLE([http3],
[AS_HELP_STRING([--enable-http3],
[(EXPERIMENTAL) Enable HTTP/3. This requires ngtcp2, nghttp3, and a custom OpenSSL.])],
[request_http3=$enableval], [request_http3=no])
AC_ARG_WITH([libxml2],
[AS_HELP_STRING([--with-libxml2],
[Use libxml2 [default=check]])],
......@@ -172,6 +177,16 @@ AC_ARG_WITH([cython],
[Use cython in given PATH])],
[cython_path=$withval], [])
AC_ARG_WITH([libngtcp2],
[AS_HELP_STRING([--with-libngtcp2],
[Use libngtcp2 [default=check]])],
[request_libngtcp2=$withval], [request_libngtcp2=check])
AC_ARG_WITH([libnghttp3],
[AS_HELP_STRING([--with-libnghttp3],
[Use libnghttp3 [default=check]])],
[request_libnghttp3=$withval], [request_libnghttp3=check])
dnl Define variables
AC_ARG_VAR([CYTHON], [the Cython executable])
......@@ -334,6 +349,13 @@ case "$host_os" in
;;
esac
case "${build}" in
*-apple-darwin*)
EXTRA_DEFS="-D__APPLE_USE_RFC_3542"
AC_SUBST([EXTRA_DEFS])
;;
esac
# zlib
have_zlib=no
if test "x${request_zlib}" != "xno"; then
......@@ -455,26 +477,50 @@ if test "x${request_libcares}" = "xyes" &&
fi
# ngtcp2 (for src)
PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.0.0], [have_libngtcp2=yes],
[have_libngtcp2=no])
if test "x${have_libngtcp2}" = "xno"; then
AC_MSG_NOTICE($LIBNGTCP2_PKG_ERRORS)
have_libngtcp2=no
if test "x${request_libngtcp2}" != "xno"; then
PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.0.0], [have_libngtcp2=yes],
[have_libngtcp2=no])
if test "x${have_libngtcp2}" = "xno"; then
AC_MSG_NOTICE($LIBNGTCP2_PKG_ERRORS)
fi
fi
if test "x${request_libngtcp2}" = "xyes" &&
test "x${have_libngtcp2}" != "xyes"; then
AC_MSG_ERROR([libngtcp2 was requested (--with-libngtcp2) but not found])
fi
# ngtcp2_crypto_openssl (for src)
PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_OPENSSL],
[libngtcp2_crypto_openssl >= 0.0.0],
[have_libngtcp2_crypto_openssl=yes],
[have_libngtcp2_crypto_openssl=no])
if test "x${have_libngtcp2_crypto_openssl}" = "xno"; then
AC_MSG_NOTICE($LIBNGTCP2_CRYPTO_OPENSSL_PKG_ERRORS)
have_libngtcp2_crypto_openssl=no
if test "x${request_libngtcp2}" != "xno"; then
PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_OPENSSL],
[libngtcp2_crypto_openssl >= 0.0.0],
[have_libngtcp2_crypto_openssl=yes],
[have_libngtcp2_crypto_openssl=no])
if test "x${have_libngtcp2_crypto_openssl}" = "xno"; then
AC_MSG_NOTICE($LIBNGTCP2_CRYPTO_OPENSSL_PKG_ERRORS)
fi
fi
if test "x${request_libngtcp2}" = "xyes" &&
test "x${have_libngtcp2_crypto_openssl}" != "xyes"; then
AC_MSG_ERROR([libngtcp2_crypto_openssl was requested (--with-libngtcp2) but not found])
fi
# nghttp3 (for src)
PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.0.0], [have_libnghttp3=yes],
[have_libnghttp3=no])
if test "x${have_libnghttp3}" = "xno"; then
AC_MSG_NOTICE($LIBNGHTTP3_PKT_ERRORS)
have_libnghttp3=no
if test "x${request_libnghttp3}" != "xno"; then
PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.0.0], [have_libnghttp3=yes],
[have_libnghttp3=no])
if test "x${have_libnghttp3}" = "xno"; then
AC_MSG_NOTICE($LIBNGHTTP3_PKG_ERRORS)
fi
fi
if test "x${request_libnghttp3}" = "xyes" &&
test "x${have_libnghttp3}" != "xyes"; then
AC_MSG_ERROR([libnghttp3 was requested (--with-libnghttp3) but not found])
fi
# libevent_openssl (for examples)
......@@ -621,6 +667,23 @@ fi
AM_CONDITIONAL([ENABLE_APP], [ test "x${enable_app}" = "xyes" ])
# Check HTTP/3 support
enable_http3=no
if test "x${request_http3}" != "xno" &&
test "x${have_libngtcp2}" = "xyes" &&
test "x${have_libngtcp2_crypto_openssl}" = "xyes" &&
test "x${have_libnghttp3}" = "xyes"; then
enable_http3=yes
AC_DEFINE([ENABLE_HTTP3], [1], [Define to 1 if HTTP/3 is enabled.])
fi
if test "x${request_http3}" = "xyes" &&
test "x${enable_http3}" != "xyes"; then
AC_MSG_ERROR([HTTP/3 was requested (--enable-http3) but dependencies are not met.])
fi
AM_CONDITIONAL([ENABLE_HTTP3], [ test "x${enable_http3}" = "xyes" ])
enable_hpack_tools=no
# HPACK tools requires jansson
if test "x${request_hpack_tools}" != "xno" &&
......@@ -1058,4 +1121,5 @@ AC_MSG_NOTICE([summary of build options:
Examples: ${enable_examples}
Python bindings:${enable_python_bindings}
Threading: ${enable_threads}
HTTP/3 (EXPERIMENTAL): ${enable_http3}
])
src/Makefile.am
View file @ 4eced8a3
......@@ -51,6 +51,7 @@ AM_CPPFLAGS = \
@LIBNGTCP2_CFLAGS@ \
@JANSSON_CFLAGS@ \
@ZLIB_CFLAGS@ \
@EXTRA_DEFS@ \
@DEFS@
AM_LDFLAGS = @LIBTOOL_LDFLAGS@
......@@ -103,10 +104,14 @@ h2load_SOURCES = util.cc util.h \
tls.cc tls.h \
h2load_session.h \
h2load_http2_session.cc h2load_http2_session.h \
h2load_http1_session.cc h2load_http1_session.h \
h2load_http1_session.cc h2load_http1_session.h
if ENABLE_HTTP3
h2load_SOURCES += \
h2load_http3_session.cc h2load_http3_session.h \
h2load_quic.cc h2load_quic.h \
quic.cc quic.h
endif # ENABLE_HTTP3
NGHTTPX_SRCS = \
util.cc util.h http2.cc http2.h timegm.c timegm.h base64.h \
......@@ -153,12 +158,6 @@ NGHTTPX_SRCS = \
shrpx_dns_resolver.cc shrpx_dns_resolver.h \
shrpx_dual_dns_resolver.cc shrpx_dual_dns_resolver.h \
shrpx_dns_tracker.cc shrpx_dns_tracker.h \
shrpx_quic.cc shrpx_quic.h \
shrpx_quic_listener.cc shrpx_quic_listener.h \
shrpx_quic_connection_handler.cc shrpx_quic_connection_handler.h \
shrpx_http3_upstream.cc shrpx_http3_upstream.h \
http3.cc http3.h \
quic.cc quic.h \
buffer.h memchunk.h template.h allocator.h \
xsi_strerror.c xsi_strerror.h
......@@ -171,6 +170,16 @@ NGHTTPX_SRCS += \
shrpx_mruby_module_response.cc shrpx_mruby_module_response.h
endif # HAVE_MRUBY
if ENABLE_HTTP3
NGHTTPX_SRCS += \
shrpx_quic.cc shrpx_quic.h \
shrpx_quic_listener.cc shrpx_quic_listener.h \
shrpx_quic_connection_handler.cc shrpx_quic_connection_handler.h \
shrpx_http3_upstream.cc shrpx_http3_upstream.h \
http3.cc http3.h \
quic.cc quic.h
endif # ENABLE_HTTP3
noinst_LIBRARIES = libnghttpx.a
libnghttpx_a_SOURCES = ${NGHTTPX_SRCS}
libnghttpx_a_CPPFLAGS = ${AM_CPPFLAGS}
......
src/h2load.cc
View file @ 4eced8a3
......@@ -50,14 +50,18 @@
#include <openssl/err.h>
#include <ngtcp2/ngtcp2.h>
#ifdef ENABLE_HTTP3
# include <ngtcp2/ngtcp2.h>
#endif // ENABLE_HTTP3
#include "url-parser/url_parser.h"
#include "h2load_http1_session.h"
#include "h2load_http2_session.h"
#include "h2load_http3_session.h"
#include "h2load_quic.h"
#ifdef ENABLE_HTTP3
# include "h2load_http3_session.h"
# include "h2load_quic.h"
#endif // ENABLE_HTTP3
#include "tls.h"
#include "http2.h"
#include "util.h"
......@@ -141,8 +145,12 @@ bool Config::is_timing_based_mode() const { return (this->duration > 0); }
bool Config::has_base_uri() const { return (!this->base_uri.empty()); }
bool Config::rps_enabled() const { return this->rps > 0.0; }
bool Config::is_quic() const {
#ifdef ENABLE_HTTP3
return !npn_list.empty() &&
(npn_list[0] == NGHTTP3_ALPN_H3 || npn_list[0] == "\x5h3-29");
#else // !ENABLE_HTTP3
return false;
#endif // !ENABLE_HTTP3
}
Config config;
......@@ -435,7 +443,9 @@ Client::Client(uint32_t id, Worker *worker, size_t req_todo)
cstat{},
worker(worker),
ssl(nullptr),
#ifdef ENABLE_HTTP3
quic{},
#endif // ENABLE_HTTP3
next_addr(config.addrs),
current_addr(nullptr),
reqidx(0),
......@@ -478,16 +488,20 @@ Client::Client(uint32_t id, Worker *worker, size_t req_todo)
ev_timer_init(&rps_watcher, rps_cb, 0., 0.);
rps_watcher.data = this;
#ifdef ENABLE_HTTP3
ev_timer_init(&quic.pkt_timer, quic_pkt_timeout_cb, 0., 0.);
quic.pkt_timer.data = this;
#endif // ENABLE_HTTP3
}
Client::~Client() {
disconnect();
#ifdef ENABLE_HTTP3
if (config.is_quic()) {
quic_free();
}
#endif // ENABLE_HTTP3
if (ssl) {
SSL_free(ssl);
......@@ -504,6 +518,7 @@ int Client::make_socket(addrinfo *addr) {
int rv;
if (config.is_quic()) {
#ifdef ENABLE_HTTP3
fd = util::create_nonblock_udp_socket(addr->ai_family);
if (fd == -1) {
return -1;
......@@ -528,6 +543,7 @@ int Client::make_socket(addrinfo *addr) {
std::cerr << "quic_init failed" << std::endl;
return -1;
}
#endif // ENABLE_HTTP3
} else {
fd = util::create_nonblock_socket(addr->ai_family);
if (fd == -1) {
......@@ -614,10 +630,12 @@ int Client::connect() {
ev_io_start(worker->loop, &wev);
if (config.is_quic()) {
#ifdef ENABLE_HTTP3
ev_io_start(worker->loop, &rev);
readfn = &Client::read_quic;
writefn = &Client::write_quic;
#endif // ENABLE_HTTP3
} else {
writefn = &Client::connected;
}
......@@ -676,11 +694,15 @@ void Client::fail() {
void Client::disconnect() {
record_client_end_time();
#ifdef ENABLE_HTTP3
if (config.is_quic()) {
quic_close_connection();
}
#endif // ENABLE_HTTP3
#ifdef ENABLE_HTTP3
ev_timer_stop(worker->loop, &quic.pkt_timer);
#endif // ENABLE_HTTP3
ev_timer_stop(worker->loop, &conn_inactivity_watcher);
ev_timer_stop(worker->loop, &conn_active_watcher);
ev_timer_stop(worker->loop, &rps_watcher);
......@@ -843,9 +865,11 @@ void Client::report_app_info() {
}
void Client::terminate_session() {
#ifdef ENABLE_HTTP3
if (config.is_quic()) {
quic.close_requested = true;
}
#endif // ENABLE_HTTP3
if (session) {
session->terminate();
}
......@@ -1047,11 +1071,13 @@ int Client::connection_made() {
if (next_proto) {
auto proto = StringRef{next_proto, next_proto_len};
if (config.is_quic()) {
#ifdef ENABLE_HTTP3
assert(session);
if (!util::streq(StringRef{&NGHTTP3_ALPN_H3[1]}, proto) &&
!util::streq_l("h3-29", proto)) {
return -1;
}
#endif // ENABLE_HTTP3
} else if (util::check_h2_is_selected(proto)) {
session = std::make_unique<Http2Session>(this);
} else if (util::streq(NGHTTP2_H1_1, proto)) {
......@@ -1377,6 +1403,7 @@ int Client::write_tls() {
return 0;
}
#ifdef ENABLE_HTTP3
int Client::write_udp(const sockaddr *addr, socklen_t addrlen,
const uint8_t *data, size_t datalen, size_t gso_size) {
iovec msg_iov;
......@@ -1389,7 +1416,7 @@ int Client::write_udp(const sockaddr *addr, socklen_t addrlen,
msg.msg_iov = &msg_iov;
msg.msg_iovlen = 1;
#ifdef UDP_SEGMENT
# ifdef UDP_SEGMENT
std::array<uint8_t, CMSG_SPACE(sizeof(uint16_t))> msg_ctrl{};
if (gso_size && datalen > gso_size) {
msg.msg_control = msg_ctrl.data();
......@@ -1401,7 +1428,7 @@ int Client::write_udp(const sockaddr *addr, socklen_t addrlen,
cm->cmsg_len = CMSG_LEN(sizeof(uint16_t));
*(reinterpret_cast<uint16_t *>(CMSG_DATA(cm))) = gso_size;
}
#endif // UDP_SEGMENT
# endif // UDP_SEGMENT
auto nwrite = sendmsg(fd, &msg, 0);
if (nwrite < 0) {
......@@ -1414,6 +1441,7 @@ int Client::write_udp(const sockaddr *addr, socklen_t addrlen,
return 0;
}
#endif // ENABLE_HTTP3
void Client::record_request_time(RequestStat *req_stat) {
req_stat->request_time = std::chrono::steady_clock::now();
......@@ -2740,7 +2768,9 @@ int main(int argc, char **argv) {
<< "Warning: --qlog-file-base: only effective in quic, ignoring."
<< std::endl;
} else {
#ifdef ENABLE_HTTP3
config.qlog_file_base = qlog_base;
#endif // ENABLE_HTTP3
}
}
......@@ -2764,8 +2794,10 @@ int main(int argc, char **argv) {
SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);
if (config.is_quic()) {
#ifdef ENABLE_HTTP3
SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION);
SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION);
#endif // ENABLE_HTTP3
} else if (nghttp2::tls::ssl_ctx_set_proto_versions(
ssl_ctx, nghttp2::tls::NGHTTP2_TLS_MIN_VERSION,
nghttp2::tls::NGHTTP2_TLS_MAX_VERSION) != 0) {
......@@ -2780,12 +2812,14 @@ int main(int argc, char **argv) {
exit(EXIT_FAILURE);
}
#if OPENSSL_1_1_1_API
if (SSL_CTX_set_ciphersuites(ssl_ctx, config.tls13_ciphers.c_str()) == 0) {
std::cerr << "SSL_CTX_set_ciphersuites with " << config.tls13_ciphers
<< " failed: " << ERR_error_string(ERR_get_error(), nullptr)
<< std::endl;
exit(EXIT_FAILURE);
}
#endif // OPENSSL_1_1_1_API
if (SSL_CTX_set1_groups_list(ssl_ctx, config.groups.c_str()) != 1) {
std::cerr << "SSL_CTX_set1_groups_list failed" << std::endl;
......@@ -2806,6 +2840,7 @@ int main(int argc, char **argv) {
SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size());
#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
#if OPENSSL_1_1_1_API
auto keylog_filename = getenv("SSLKEYLOGFILE");
if (keylog_filename) {
keylog_file.open(keylog_filename, std::ios_base::app);
......@@ -2813,6 +2848,7 @@ int main(int argc, char **argv) {
SSL_CTX_set_keylog_callback(ssl_ctx, keylog_callback);
}
}
#endif // OPENSSL_1_1_1_API
std::string user_agent = "h2load nghttp2/" NGHTTP2_VERSION;
Headers shared_nva;
......@@ -3093,10 +3129,12 @@ traffic: )" << util::utos_funit(stats.bytes_total)
<< ") headers (space savings " << header_space_savings * 100
<< "%), " << util::utos_funit(stats.bytes_body) << "B ("
<< stats.bytes_body << R"() data)" << std::endl;
#ifdef ENABLE_HTTP3
if (config.is_quic()) {
std::cout << "UDP datagram: " << stats.udp_dgram_sent << " sent, "
<< stats.udp_dgram_recv << " received" << std::endl;
}
#endif // ENABLE_HTTP3
std::cout
<< R"( min max mean sd +/- sd
time for request: )"
......
src/h2load.h
View file @ 4eced8a3
......@@ -45,15 +45,19 @@
#include <nghttp2/nghttp2.h>
#include <ngtcp2/ngtcp2.h>
#include <ngtcp2/ngtcp2_crypto.h>
#ifdef ENABLE_HTTP3
# include <ngtcp2/ngtcp2.h>
# include <ngtcp2/ngtcp2_crypto.h>
#endif // ENABLE_HTTP3
#include <ev.h>
#include <openssl/ssl.h>
#include "http2.h"
#include "quic.h"
#ifdef ENABLE_HTTP3
# include "quic.h"
#endif // ENABLE_HTTP3
#include "memchunk.h"
#include "template.h"
......@@ -327,6 +331,7 @@ struct Client {
std::function<int(Client &)> readfn, writefn;
Worker *worker;
SSL *ssl;
#ifdef ENABLE_HTTP3
struct {
ev_timer pkt_timer;
ngtcp2_conn *conn;
......@@ -335,6 +340,7 @@ struct Client {
bool close_requested;
FILE *qlog_file;
} quic;
#endif // ENABLE_HTTP3
ev_timer request_timeout_watcher;
addrinfo *next_addr;
// Address for the current address. When try_new_connection() is
......@@ -447,6 +453,7 @@ struct Client {
void signal_write();
#ifdef ENABLE_HTTP3
// QUIC
int quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
const sockaddr *remote_addr, socklen_t remote_addrlen);
......@@ -475,6 +482,7 @@ struct Client {
int quic_pkt_timeout();
void quic_restart_pkt_timer();
void quic_write_qlog(const void *data, size_t datalen);
#endif // ENABLE_HTTP3
};
} // namespace h2load
......
src/shrpx.cc
View file @ 4eced8a3
......@@ -1548,6 +1548,7 @@ void fill_default_config(Config *config) {
downstreamconf.option, downstreamconf.encoder_dynamic_table_size);
}
#ifdef ENABLE_HTTP3
auto &quicconf = config->quic;
{
quicconf.timeout.idle = 30_s;
......@@ -1560,6 +1561,7 @@ void fill_default_config(Config *config) {
exit(EXIT_FAILURE);
}
}
#endif // ENABLE_HTTP3
auto &loggingconf = config->logging;
{
......
src/shrpx_client_handler.cc
View file @ 4eced8a3
......@@ -51,7 +51,9 @@
#include "shrpx_api_downstream_connection.h"
#include "shrpx_health_monitor_downstream_connection.h"
#include "shrpx_null_downstream_connection.h"
#include "shrpx_http3_upstream.h"
#ifdef ENABLE_HTTP3
# include "shrpx_http3_upstream.h"
#endif // ENABLE_HTTP3
#include "shrpx_log.h"
#include "util.h"
#include "template.h"
......@@ -287,6 +289,7 @@ int ClientHandler::write_tls() {
}
}
#ifdef ENABLE_HTTP3
int ClientHandler::read_quic(const UpstreamAddr *faddr,
const Address &remote_addr,
const Address &local_addr, const uint8_t *data,
......@@ -297,6 +300,7 @@ int ClientHandler::read_quic(const UpstreamAddr *faddr,
}
int ClientHandler::write_quic() { return upstream_->on_write(); }
#endif // ENABLE_HTTP3
int ClientHandler::upstream_noop() { return 0; }
......@@ -509,12 +513,14 @@ void ClientHandler::setup_upstream_io_callback() {
}
}
#ifdef ENABLE_HTTP3
void ClientHandler::setup_http3_upstream(
std::unique_ptr<Http3Upstream> &&upstream) {
upstream_ = std::move(upstream);
alpn_ = StringRef::from_lit("h3");
write_ = &ClientHandler::write_quic;
}
#endif // ENABLE_HTTP3
ClientHandler::~ClientHandler() {
if (LOG_ENABLED(INFO)) {
......
src/shrpx_client_handler.h
View file @ 4eced8a3
......@@ -53,7 +53,9 @@ class Downstream;
struct WorkerStat;
struct DownstreamAddrGroup;
struct DownstreamAddr;
#ifdef ENABLE_HTTP3
class Http3Upstream;
#endif // ENABLE_HTTP3
class ClientHandler {
public:
......@@ -71,9 +73,6 @@ public:
int read_tls();
int write_tls();
int read_quic(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const uint8_t *data, size_t datalen);
int upstream_noop();
int upstream_read();
int upstream_http2_connhd_read();
......@@ -147,8 +146,12 @@ public:
void setup_upstream_io_callback();
#ifdef ENABLE_HTTP3
void setup_http3_upstream(std::unique_ptr<Http3Upstream> &&upstream);
int read_quic(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const uint8_t *data, size_t datalen);
int write_quic();
#endif // ENABLE_HTTP3
// Returns string suitable for use in "by" parameter of Forwarded
// header field.
......
src/shrpx_config.cc
View file @ 4eced8a3
......@@ -821,7 +821,11 @@ int parse_upstream_params(UpstreamParams &out, const StringRef &src_params) {
} else if (util::strieq_l("proxyproto", param)) {
out.proxyproto = true;
} else if (util::strieq_l("quic", param)) {
#ifdef ENABLE_HTTP3
out.quic = true;
#else // !ENABLE_HTTP3
LOG(ERROR) << "quic: QUIC is disabled at compile time";
#endif // !ENABLE_HTTP3
} else if (!param.empty()) {
LOG(ERROR) << "frontend: " << param << ": unknown keyword";
return -1;
......@@ -2674,8 +2678,12 @@ int parse_config(Config *config, int optid, const StringRef &opt,
apiconf.enabled = true;
}
#ifdef ENABLE_HTTP3
auto &addrs = params.quic ? config->conn.quic_listener.addrs
: config->conn.listener.addrs;
#else // !ENABLE_HTTP3
auto &addrs = config->conn.listener.addrs;
#endif // !ENABLE_HTTP3
if (util::istarts_with(optarg, SHRPX_UNIX_PATH_PREFIX)) {
auto path = std::begin(optarg) + SHRPX_UNIX_PATH_PREFIX.size();
......
src/shrpx_config.h
View file @ 4eced8a3
......@@ -704,6 +704,7 @@ struct TLSConfig {
bool no_postpone_early_data;
};
#ifdef ENABLE_HTTP3
struct QUICConfig {
struct {
std::array<uint8_t, 32> secret;
......@@ -715,6 +716,7 @@ struct QUICConfig {
bool log;
} debug;
};
#endif // ENABLE_HTTP3
// custom error page
struct ErrorPage {
......@@ -921,9 +923,11 @@ struct ConnectionConfig {
int fastopen;
} listener;
#ifdef ENABLE_HTTP3
struct {
std::vector<UpstreamAddr> addrs;
} quic_listener;
#endif // ENABLE_HTTP3
struct {
struct {
......@@ -968,7 +972,9 @@ struct Config {
http{},
http2{},
tls{},
#ifdef ENABLE_HTTP3
quic{},
#endif // ENABLE_HTTP3
logging{},
conn{},
api{},
......@@ -986,7 +992,8 @@ struct Config {
single_process{false},
single_thread{false},
ignore_per_pattern_mruby_error{false},
ev_loop_flags{0} {}
ev_loop_flags{0} {
}
~Config();
Config(Config &&) = delete;
......@@ -1002,7 +1009,9 @@ struct Config {
HttpConfig http;
Http2Config http2;
TLSConfig tls;
#ifdef ENABLE_HTTP3
QUICConfig quic;
#endif // ENABLE_HTTP3
LoggingConfig logging;
ConnectionConfig conn;
APIConfig api;
......
src/shrpx_connection_handler.cc
View file @ 4eced8a3
......@@ -156,6 +156,7 @@ ConnectionHandler::~ConnectionHandler() {
ev_timer_stop(loop_, &ocsp_timer_);
ev_timer_stop(loop_, &disable_acceptor_timer_);
#ifdef ENABLE_HTTP3
for (auto ssl_ctx : quic_all_ssl_ctx_) {
if (ssl_ctx == nullptr) {
continue;
......@@ -166,6 +167,7 @@ ConnectionHandler::~ConnectionHandler() {
delete tls_ctx_data;
SSL_CTX_free(ssl_ctx);
}
#endif // ENABLE_HTTP3
for (auto ssl_ctx : all_ssl_ctx_) {
auto tls_ctx_data =
......@@ -221,14 +223,16 @@ int ConnectionHandler::create_single_worker() {
#endif // HAVE_NEVERBLEED
);
#ifdef ENABLE_HTTP3
quic_cert_tree_ = tls::create_cert_lookup_tree();
auto quic_sv_ssl_ctx = tls::setup_quic_server_ssl_context(
quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, quic_cert_tree_.get()
#ifdef HAVE_NEVERBLEED
# ifdef HAVE_NEVERBLEED
,
nb_
#endif // HAVE_NEVERBLEED
# endif // HAVE_NEVERBLEED
);
#endif // ENABLE_HTTP3
auto cl_ssl_ctx = tls::setup_downstream_client_ssl_context(
#ifdef HAVE_NEVERBLEED
......@@ -238,7 +242,9 @@ int ConnectionHandler::create_single_worker() {
if (cl_ssl_ctx) {
all_ssl_ctx_.push_back(cl_ssl_ctx);
#ifdef ENABLE_HTTP3
quic_all_ssl_ctx_.push_back(nullptr);
#endif // ENABLE_HTTP3
}
auto config = get_config();
......@@ -255,22 +261,29 @@ int ConnectionHandler::create_single_worker() {
tlsconf.cacert, memcachedconf.cert_file,
memcachedconf.private_key_file, nullptr);
all_ssl_ctx_.push_back(session_cache_ssl_ctx);
#ifdef ENABLE_HTTP3
quic_all_ssl_ctx_.push_back(nullptr);
#endif // ENABLE_HTTP3
}
}
single_worker_ = std::make_unique<Worker>(
loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
quic_sv_ssl_ctx, quic_cert_tree_.get(), ticket_keys_, this,
config->conn.downstream);
#ifdef ENABLE_HTTP3
quic_sv_ssl_ctx, quic_cert_tree_.get(),
#endif // ENABLE_HTTP3
ticket_keys_, this, config->conn.downstream);
#ifdef HAVE_MRUBY
if (single_worker_->create_mruby_context() != 0) {
return -1;
}
#endif // HAVE_MRUBY
#ifdef ENABLE_HTTP3
if (single_worker_->setup_quic_server_socket() != 0) {
return -1;
}
#endif // ENABLE_HTTP3
return 0;
}
......@@ -288,14 +301,16 @@ int ConnectionHandler::create_worker_thread(size_t num) {
# endif // HAVE_NEVERBLEED
);
# ifdef ENABLE_HTTP3
quic_cert_tree_ = tls::create_cert_lookup_tree();
auto quic_sv_ssl_ctx = tls::setup_quic_server_ssl_context(
quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, quic_cert_tree_.get()
# ifdef HAVE_NEVERBLEED
# ifdef HAVE_NEVERBLEED
,
nb_
# endif // HAVE_NEVERBLEED
# endif // HAVE_NEVERBLEED
);
# endif // ENABLE_HTTP3
auto cl_ssl_ctx = tls::setup_downstream_client_ssl_context(
# ifdef HAVE_NEVERBLEED
......@@ -305,7 +320,9 @@ int ConnectionHandler::create_worker_thread(size_t num) {
if (cl_ssl_ctx) {
all_ssl_ctx_.push_back(cl_ssl_ctx);
# ifdef ENABLE_HTTP3
quic_all_ssl_ctx_.push_back(nullptr);
# endif // ENABLE_HTTP3
}
auto config = get_config();
......@@ -329,7 +346,9 @@ int ConnectionHandler::create_worker_thread(size_t num) {
tlsconf.cacert, memcachedconf.cert_file,
memcachedconf.private_key_file, nullptr);
all_ssl_ctx_.push_back(session_cache_ssl_ctx);
# ifdef ENABLE_HTTP3
quic_all_ssl_ctx_.push_back(nullptr);
# endif // ENABLE_HTTP3
}
}
......@@ -338,17 +357,22 @@ int ConnectionHandler::create_worker_thread(size_t num) {
auto worker = std::make_unique<Worker>(
loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
quic_sv_ssl_ctx, quic_cert_tree_.get(), ticket_keys_, this,
config->conn.downstream);
# ifdef ENABLE_HTTP3
quic_sv_ssl_ctx, quic_cert_tree_.get(),
# endif // ENABLE_HTTP3
ticket_keys_, this, config->conn.downstream);
# ifdef HAVE_MRUBY
if (worker->create_mruby_context() != 0) {
return -1;
}
# endif // HAVE_MRUBY
# ifdef ENABLE_HTTP3
if ((!apiconf.enabled || i != 0) &&
worker->setup_quic_server_socket() != 0) {
return -1;
}
# endif // ENABLE_HTTP3
workers_.push_back(std::move(worker));
worker_loops_.push_back(loop);
......@@ -646,7 +670,9 @@ void ConnectionHandler::handle_ocsp_complete() {
ev_child_stop(loop_, &ocsp_.chldev);
assert(ocsp_.next < all_ssl_ctx_.size());
#ifdef ENABLE_HTTP3
assert(all_ssl_ctx_.size() == quic_all_ssl_ctx_.size());
#endif // ENABLE_HTTP3
auto ssl_ctx = all_ssl_ctx_[ocsp_.next];
auto tls_ctx_data =
......@@ -674,6 +700,7 @@ void ConnectionHandler::handle_ocsp_complete() {
if (tlsconf.ocsp.no_verify ||
tls::verify_ocsp_response(ssl_ctx, ocsp_.resp.data(),
ocsp_.resp.size()) == 0) {
#ifdef ENABLE_HTTP3
// We have list of SSL_CTX with the same certificate in
// quic_all_ssl_ctx_ as well. Some SSL_CTXs are missing there in
// that case we get nullptr.
......@@ -681,21 +708,22 @@ void ConnectionHandler::handle_ocsp_complete() {
if (quic_ssl_ctx) {
auto quic_tls_ctx_data = static_cast<tls::TLSContextData *>(
SSL_CTX_get_app_data(quic_ssl_ctx));
#ifndef OPENSSL_IS_BORINGSSL
# ifdef HAVE_ATOMIC_STD_SHARED_PTR
# ifndef OPENSSL_IS_BORINGSSL
# ifdef HAVE_ATOMIC_STD_SHARED_PTR
std::atomic_store_explicit(
&quic_tls_ctx_data->ocsp_data,
std::make_shared<std::vector<uint8_t>>(ocsp_.resp),
std::memory_order_release);
# else // !HAVE_ATOMIC_STD_SHARED_PTR
# else // !HAVE_ATOMIC_STD_SHARED_PTR
std::lock_guard<std::mutex> g(quic_tls_ctx_data->mu);
quic_tls_ctx_data->ocsp_data =
std::make_shared<std::vector<uint8_t>>(ocsp_.resp);
# endif // !HAVE_ATOMIC_STD_SHARED_PTR
#else // OPENSSL_IS_BORINGSSL
# endif // !HAVE_ATOMIC_STD_SHARED_PTR
# else // OPENSSL_IS_BORINGSSL
SSL_CTX_set_ocsp_response(ssl_ctx, ocsp_.resp.data(), ocsp_.resp.size());
#endif // OPENSSL_IS_BORINGSSL
# endif // OPENSSL_IS_BORINGSSL
}
#endif // ENABLE_HTTP3
#ifndef OPENSSL_IS_BORINGSSL
# ifdef HAVE_ATOMIC_STD_SHARED_PTR
......@@ -877,7 +905,9 @@ SSL_CTX *ConnectionHandler::create_tls_ticket_key_memcached_ssl_ctx() {
nullptr);
all_ssl_ctx_.push_back(ssl_ctx);
#ifdef ENABLE_HTTP3
quic_all_ssl_ctx_.push_back(nullptr);
#endif // ENABLE_HTTP3
return ssl_ctx;
}
......@@ -940,10 +970,12 @@ ConnectionHandler::get_indexed_ssl_ctx(size_t idx) const {
return indexed_ssl_ctx_[idx];
}
#ifdef ENABLE_HTTP3
const std::vector<SSL_CTX *> &
ConnectionHandler::get_quic_indexed_ssl_ctx(size_t idx) const {
return quic_indexed_ssl_ctx_[idx];
}
#endif // ENABLE_HTTP3
void ConnectionHandler::set_enable_acceptor_on_ocsp_completion(bool f) {
enable_acceptor_on_ocsp_completion_ = f;
......
src/shrpx_connection_handler.h
View file @ 4eced8a3
......@@ -159,7 +159,9 @@ public:
SSL_CTX *get_ssl_ctx(size_t idx) const;
const std::vector<SSL_CTX *> &get_indexed_ssl_ctx(size_t idx) const;
#ifdef ENABLE_HTTP3
const std::vector<SSL_CTX *> &get_quic_indexed_ssl_ctx(size_t idx) const;
#endif // ENABLE_HTTP3
#ifdef HAVE_NEVERBLEED
void set_neverbleed(neverbleed_t *nb);
......@@ -188,8 +190,10 @@ private:
// selection among them are performed by hostname presented by SNI,
// and signature algorithm presented by client.
std::vector<std::vector<SSL_CTX *>> indexed_ssl_ctx_;
#ifdef ENABLE_HTTP3
std::vector<SSL_CTX *> quic_all_ssl_ctx_;
std::vector<std::vector<SSL_CTX *>> quic_indexed_ssl_ctx_;
#endif // ENABLE_HTTP3
OCSPUpdateContext ocsp_;
std::mt19937 &gen_;
// ev_loop for each worker
......@@ -206,7 +210,9 @@ private:
// Otherwise, nullptr and workers_ has instances of Worker instead.
std::unique_ptr<Worker> single_worker_;
std::unique_ptr<tls::CertLookupTree> cert_tree_;
#ifdef ENABLE_HTTP3
std::unique_ptr<tls::CertLookupTree> quic_cert_tree_;
#endif // ENABLE_HTTP3
std::unique_ptr<MemcachedDispatcher> tls_ticket_key_memcached_dispatcher_;
// Current TLS session ticket keys. Note that TLS connection does
// not refer to this field directly. They use TicketKeys object in
......
src/shrpx_downstream.cc
View file @ 4eced8a3
......@@ -165,7 +165,9 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool,
downstream_wtimer_.data = this;
rcbufs_.reserve(32);
#ifdef ENABLE_HTTP3
rcbufs3_.reserve(32);
#endif // ENABLE_HTTP3
}
Downstream::~Downstream() {
......@@ -205,9 +207,11 @@ Downstream::~Downstream() {
// explicitly.
dconn_.reset();
#ifdef ENABLE_HTTP3
for (auto rcbuf : rcbufs3_) {
nghttp3_rcbuf_decref(rcbuf);
}
#endif // ENABLE_HTTP3
for (auto rcbuf : rcbufs_) {
nghttp2_rcbuf_decref(rcbuf);
......@@ -1134,10 +1138,12 @@ void Downstream::add_rcbuf(nghttp2_rcbuf *rcbuf) {
rcbufs_.push_back(rcbuf);
}
#ifdef ENABLE_HTTP3
void Downstream::add_rcbuf(nghttp3_rcbuf *rcbuf) {
nghttp3_rcbuf_incref(rcbuf);
rcbufs3_.push_back(rcbuf);
}
#endif // ENABLE_HTTP3
void Downstream::set_downstream_addr_group(
const std::shared_ptr<DownstreamAddrGroup> &group) {
......
src/shrpx_downstream.h
View file @ 4eced8a3
......@@ -38,7 +38,9 @@
#include <nghttp2/nghttp2.h>
#include <nghttp3/nghttp3.h>
#ifdef ENABLE_HTTP3
# include <nghttp3/nghttp3.h>
#endif // ENABLE_HTTP3
#include "llhttp.h"
......@@ -490,7 +492,9 @@ public:
BlockAllocator &get_block_allocator();
void add_rcbuf(nghttp2_rcbuf *rcbuf);
#ifdef ENABLE_HTTP3
void add_rcbuf(nghttp3_rcbuf *rcbuf);
#endif // ENABLE_HTTP3
void
set_downstream_addr_group(const std::shared_ptr<DownstreamAddrGroup> &group);
......@@ -533,7 +537,9 @@ private:
BlockAllocator balloc_;
std::vector<nghttp2_rcbuf *> rcbufs_;
#ifdef ENABLE_HTTP3
std::vector<nghttp3_rcbuf *> rcbufs3_;
#endif // ENABLE_HTTP3
Request req_;
Response resp_;
......
src/shrpx_tls.cc
View file @ 4eced8a3
This diff is collapsed.
src/shrpx_tls.h
View file @ 4eced8a3
......@@ -100,15 +100,17 @@ SSL_CTX *create_ssl_client_context(
unsigned char *outlen, const unsigned char *in,
unsigned int inlen, void *arg));
#ifdef ENABLE_HTTP3
SSL_CTX *create_quic_ssl_client_context(
#ifdef HAVE_NEVERBLEED
# ifdef HAVE_NEVERBLEED
neverbleed_t *nb,
#endif // HAVE_NEVERBLEED
# endif // HAVE_NEVERBLEED
const StringRef &cacert, const StringRef &cert_file,
const StringRef &private_key_file,
int (*next_proto_select_cb)(SSL *s, unsigned char **out,
unsigned char *outlen, const unsigned char *in,
unsigned int inlen, void *arg));
#endif // ENABLE_HTTP3
ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr,
int addrlen, const UpstreamAddr *faddr);
......@@ -227,15 +229,17 @@ setup_server_ssl_context(std::vector<SSL_CTX *> &all_ssl_ctx,
#endif // HAVE_NEVERBLEED
);
#ifdef ENABLE_HTTP3
SSL_CTX *setup_quic_server_ssl_context(
std::vector<SSL_CTX *> &all_ssl_ctx,
std::vector<std::vector<SSL_CTX *>> &indexed_ssl_ctx,
CertLookupTree *cert_tree
#ifdef HAVE_NEVERBLEED
# ifdef HAVE_NEVERBLEED
,
neverbleed_t *nb
#endif // HAVE_NEVERBLEED
# endif // HAVE_NEVERBLEED
);
#endif // ENABLE_HTTP3
// Setups client side SSL_CTX.
SSL_CTX *setup_downstream_client_ssl_context(
......
src/shrpx_worker.cc
View file @ 4eced8a3
......@@ -39,8 +39,10 @@
#ifdef HAVE_MRUBY
# include "shrpx_mruby.h"
#endif // HAVE_MRUBY
#include "shrpx_quic.h"
#include "shrpx_quic_listener.h"
#ifdef ENABLE_HTTP3
# include "shrpx_quic.h"
# include "shrpx_quic_listener.h"
#endif // ENABLE_HTTP3
#include "util.h"
#include "template.h"
......@@ -132,23 +134,29 @@ create_downstream_key(const std::shared_ptr<SharedDownstreamAddr> &shared_addr,
Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
SSL_CTX *tls_session_cache_memcached_ssl_ctx,
tls::CertLookupTree *cert_tree, SSL_CTX *quic_sv_ssl_ctx,
tls::CertLookupTree *quic_cert_tree,
tls::CertLookupTree *cert_tree,
#ifdef ENABLE_HTTP3
SSL_CTX *quic_sv_ssl_ctx, tls::CertLookupTree *quic_cert_tree,
#endif // ENABLE_HTTP3
const std::shared_ptr<TicketKeys> &ticket_keys,
ConnectionHandler *conn_handler,
std::shared_ptr<DownstreamConfig> downstreamconf)
: randgen_(util::make_mt19937()),
worker_stat_{},
dns_tracker_(loop),
#ifdef ENABLE_HTTP3
quic_upstream_addrs_{get_config()->conn.quic_listener.addrs},
#endif // ENABLE_HTTP3
loop_(loop),
sv_ssl_ctx_(sv_ssl_ctx),
cl_ssl_ctx_(cl_ssl_ctx),
cert_tree_(cert_tree),
conn_handler_(conn_handler),
#ifdef ENABLE_HTTP3
quic_sv_ssl_ctx_{quic_sv_ssl_ctx},
quic_cert_tree_{quic_cert_tree},
quic_conn_handler_{this},
#endif // ENABLE_HTTP3
ticket_keys_(ticket_keys),
connect_blocker_(
std::make_unique<ConnectBlocker>(randgen_, loop_, nullptr, nullptr)),
......@@ -511,9 +519,11 @@ void Worker::process_events() {
tls::CertLookupTree *Worker::get_cert_lookup_tree() const { return cert_tree_; }
#ifdef ENABLE_HTTP3
tls::CertLookupTree *Worker::get_quic_cert_lookup_tree() const {
return quic_cert_tree_;
}
#endif // ENABLE_HTTP3
std::shared_ptr<TicketKeys> Worker::get_ticket_keys() {
#ifdef HAVE_ATOMIC_STD_SHARED_PTR
......@@ -545,7 +555,9 @@ SSL_CTX *Worker::get_sv_ssl_ctx() const { return sv_ssl_ctx_; }
SSL_CTX *Worker::get_cl_ssl_ctx() const { return cl_ssl_ctx_; }
#ifdef ENABLE_HTTP3
SSL_CTX *Worker::get_quic_sv_ssl_ctx() const { return quic_sv_ssl_ctx_; }
#endif // ENABLE_HTTP3
void Worker::set_graceful_shutdown(bool f) { graceful_shutdown_ = f; }
......@@ -591,12 +603,15 @@ ConnectionHandler *Worker::get_connection_handler() const {
return conn_handler_;
}
#ifdef ENABLE_HTTP3
QUICConnectionHandler *Worker::get_quic_connection_handler() {
return &quic_conn_handler_;
}
#endif // ENABLE_HTTP3
DNSTracker *Worker::get_dns_tracker() { return &dns_tracker_; }
#ifdef ENABLE_HTTP3
int Worker::setup_quic_server_socket() {
for (auto &addr : quic_upstream_addrs_) {
assert(!addr.host_unix);
......@@ -609,6 +624,7 @@ int Worker::setup_quic_server_socket() {
return 0;
}
#endif // ENABLE_HTTP3
namespace {
size_t match_downstream_addr_group_host(
......
src/shrpx_worker.h
View file @ 4eced8a3
......@@ -50,7 +50,9 @@
#include "shrpx_live_check.h"
#include "shrpx_connect_blocker.h"
#include "shrpx_dns_tracker.h"
#include "shrpx_quic_connection_handler.h"
#ifdef ENABLE_HTTP3
# include "shrpx_quic_connection_handler.h"
#endif // ENABLE_HTTP3
#include "allocator.h"
using namespace nghttp2;
......@@ -62,7 +64,9 @@ class ConnectBlocker;
class MemcachedDispatcher;
struct UpstreamAddr;
class ConnectionHandler;
#ifdef ENABLE_HTTP3
class QUICListener;
#endif // ENABLE_HTTP3
#ifdef HAVE_MRUBY
namespace mruby {
......@@ -271,8 +275,10 @@ class Worker {
public:
Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
SSL_CTX *tls_session_cache_memcached_ssl_ctx,
tls::CertLookupTree *cert_tree, SSL_CTX *quic_sv_ssl_ctx,
tls::CertLookupTree *quic_cert_tree,
tls::CertLookupTree *cert_tree,
#ifdef ENABLE_HTTP3
SSL_CTX *quic_sv_ssl_ctx, tls::CertLookupTree *quic_cert_tree,
#endif // ENABLE_HTTP3
const std::shared_ptr<TicketKeys> &ticket_keys,
ConnectionHandler *conn_handler,
std::shared_ptr<DownstreamConfig> downstreamconf);
......@@ -283,7 +289,9 @@ public:
void send(const WorkerEvent &event);
tls::CertLookupTree *get_cert_lookup_tree() const;
#ifdef ENABLE_HTTP3
tls::CertLookupTree *get_quic_cert_lookup_tree() const;
#endif // ENABLE_HTTP3
// These 2 functions make a lock m_ to get/set ticket keys
// atomically.
......@@ -294,7 +302,9 @@ public:
struct ev_loop *get_loop() const;
SSL_CTX *get_sv_ssl_ctx() const;
SSL_CTX *get_cl_ssl_ctx() const;
#ifdef ENABLE_HTTP3
SSL_CTX *get_quic_sv_ssl_ctx() const;
#endif // ENABLE_HTTP3
void set_graceful_shutdown(bool f);
bool get_graceful_shutdown() const;
......@@ -324,11 +334,13 @@ public:
ConnectionHandler *get_connection_handler() const;
#ifdef ENABLE_HTTP3
QUICConnectionHandler *get_quic_connection_handler();
DNSTracker *get_dns_tracker();
int setup_quic_server_socket();
#endif // ENABLE_HTTP3
DNSTracker *get_dns_tracker();
private:
#ifndef NOTHREADS
......@@ -344,8 +356,10 @@ private:
WorkerStat worker_stat_;
DNSTracker dns_tracker_;
#ifdef ENABLE_HTTP3
std::vector<UpstreamAddr> quic_upstream_addrs_;
std::vector<std::unique_ptr<QUICListener>> quic_listeners_;
#endif // ENABLE_HTTP3
std::shared_ptr<DownstreamConfig> downstreamconf_;
std::unique_ptr<MemcachedDispatcher> session_cache_memcached_dispatcher_;
......@@ -360,10 +374,12 @@ private:
SSL_CTX *cl_ssl_ctx_;
tls::CertLookupTree *cert_tree_;
ConnectionHandler *conn_handler_;
#ifdef ENABLE_HTTP3
SSL_CTX *quic_sv_ssl_ctx_;
tls::CertLookupTree *quic_cert_tree_;
QUICConnectionHandler quic_conn_handler_;
#endif // ENABLE_HTTP3
#ifndef HAVE_ATOMIC_STD_SHARED_PTR
std::mutex ticket_keys_m_;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment