https://github.com/h2o/h2o/pull/3293 From 770208bbe3955c47e005a1e8cb08266e4a8dfc9a Mon Sep 17 00:00:00 2001 From: Remi Gacogne Date: Tue, 10 Oct 2023 15:47:57 +0200 Subject: [PATCH] [http2] delay processing requests upon observing suspicious behavior Backport of 94fbc54b6c9309912fe3d53e7b63408bbe9a1b0d to v2.2.x --- include/h2o.h | 8 +++++++ include/h2o/http2_internal.h | 8 +++++++ lib/core/config.c | 1 + lib/core/configurator.c | 9 ++++++++ lib/core/context.c | 2 ++ lib/http2/connection.c | 41 ++++++++++++++++++++++++++++++++---- 6 files changed, 65 insertions(+), 4 deletions(-) diff --git a/include/h2o.h b/include/h2o.h index 57877bd12c..409cd5c21c 100644 --- a/include/h2o.h +++ b/include/h2o.h @@ -378,6 +378,10 @@ struct st_h2o_globalconf_t { * list of callbacks */ h2o_protocol_callbacks_t callbacks; + /** + * milliseconds to delay processing requests when suspicious behavior is detected + */ + uint64_t dos_delay; } http2; struct { @@ -590,6 +594,10 @@ struct st_h2o_context_t { * timeout entry used for graceful shutdown */ h2o_timeout_entry_t _graceful_shutdown_timeout; + /* + * dos timeout + */ + h2o_timeout_t dos_delay_timeout; struct { /** * counter for http2 errors internally emitted by h2o diff --git a/include/h2o/http2_internal.h b/include/h2o/http2_internal.h index 5cfc4d8204..b9cf400929 100644 --- a/include/h2o/http2_internal.h +++ b/include/h2o/http2_internal.h @@ -179,6 +179,7 @@ struct st_h2o_http2_stream_t { h2o_linklist_t link; h2o_http2_scheduler_openref_t scheduler; } _refs; + unsigned reset_by_peer : 1; h2o_send_state_t send_state; /* state of the ostream, only used in push mode */ /* placed at last since it is large and has it's own ctor */ h2o_req_t req; @@ -232,6 +233,13 @@ struct st_h2o_http2_conn_t { } _write; h2o_cache_t *push_memo; h2o_http2_casper_t *casper; + /** + * DoS mitigation; the idea here is to delay processing requests when observing suspicious behavior + */ + struct { + h2o_timeout_entry_t process_delay; + size_t reset_budget; /* RST_STREAM frames are considered suspicious when this value goes down to zero */ + } dos_mitigation; }; int h2o_http2_update_peer_settings(h2o_http2_settings_t *settings, const uint8_t *src, size_t len, const char **err_desc); diff --git a/lib/core/config.c b/lib/core/config.c index ce1d320183..08e43a6d30 100644 --- a/lib/core/config.c +++ b/lib/core/config.c @@ -189,6 +189,7 @@ void h2o_config_init(h2o_globalconf_t *config) config->http2.latency_optimization.min_rtt = 50; // milliseconds config->http2.latency_optimization.max_additional_delay = 10; config->http2.latency_optimization.max_cwnd = 65535; + config->http2.dos_delay = 100; /* 100ms processing delay when observing suspicious behavior */ config->http2.callbacks = H2O_HTTP2_CALLBACKS; config->mimemap = h2o_mimemap_create(); diff --git a/lib/core/configurator.c b/lib/core/configurator.c index 891770cc2d..4731ba2707 100644 --- a/lib/core/configurator.c +++ b/lib/core/configurator.c @@ -531,6 +531,12 @@ static int on_config_http2_casper(h2o_configurator_command_t *cmd, h2o_configura return 0; } + +static int on_config_http2_dos_delay(h2o_configurator_command_t *cmd, h2o_configurator_context_t *ctx, yoml_t *node) +{ + return config_timeout(cmd, node, &ctx->globalconf->http2.dos_delay); +} + static int assert_is_mimetype(h2o_configurator_command_t *cmd, yoml_t *node) { if (node->type != YOML_TYPE_SCALAR) { @@ -910,6 +916,9 @@ void h2o_configurator__init_core(h2o_globalconf_t *conf) on_config_http2_push_preload); h2o_configurator_define_command(&c->super, "http2-casper", H2O_CONFIGURATOR_FLAG_GLOBAL | H2O_CONFIGURATOR_FLAG_HOST, on_config_http2_casper); + h2o_configurator_define_command(&c->super, "http2-dos-delay", + H2O_CONFIGURATOR_FLAG_GLOBAL | H2O_CONFIGURATOR_FLAG_EXPECT_SCALAR, + on_config_http2_dos_delay); h2o_configurator_define_command(&c->super, "file.mime.settypes", (H2O_CONFIGURATOR_FLAG_ALL_LEVELS & ~H2O_CONFIGURATOR_FLAG_EXTENSION) | H2O_CONFIGURATOR_FLAG_EXPECT_MAPPING, diff --git a/lib/core/context.c b/lib/core/context.c index 8d11013810..ac4b0aaf08 100644 --- a/lib/core/context.c +++ b/lib/core/context.c @@ -101,6 +101,7 @@ void h2o_context_init(h2o_context_t *ctx, h2o_loop_t *loop, h2o_globalconf_t *co h2o_linklist_init_anchor(&ctx->http1._conns); h2o_timeout_init(ctx->loop, &ctx->http2.idle_timeout, config->http2.idle_timeout); h2o_timeout_init(ctx->loop, &ctx->http2.graceful_shutdown_timeout, config->http2.graceful_shutdown_timeout); + h2o_timeout_init(ctx->loop, &ctx->http2.dos_delay_timeout, config->http2.dos_delay); h2o_linklist_init_anchor(&ctx->http2._conns); ctx->proxy.client_ctx.loop = loop; h2o_timeout_init(ctx->loop, &ctx->proxy.io_timeout, config->proxy.io_timeout); @@ -146,6 +147,7 @@ void h2o_context_dispose(h2o_context_t *ctx) h2o_timeout_dispose(ctx->loop, &ctx->http1.req_timeout); h2o_timeout_dispose(ctx->loop, &ctx->http2.idle_timeout); h2o_timeout_dispose(ctx->loop, &ctx->http2.graceful_shutdown_timeout); + h2o_timeout_dispose(ctx->loop, &ctx->http2.dos_delay_timeout); h2o_timeout_dispose(ctx->loop, &ctx->proxy.io_timeout); /* what should we do here? assert(!h2o_linklist_is_empty(&ctx->http2._conns); */ diff --git a/lib/http2/connection.c b/lib/http2/connection.c index e2da293043..4910e33098 100644 --- a/lib/http2/connection.c +++ b/lib/http2/connection.c @@ -161,7 +161,6 @@ static void update_idle_timeout(h2o_http2_conn_t *conn) h2o_timeout_unlink(&conn->_timeout_entry); if (conn->num_streams.pull.half_closed + conn->num_streams.push.half_closed == 0) { - assert(h2o_linklist_is_empty(&conn->_pending_reqs)); conn->_timeout_entry.cb = on_idle_timeout; h2o_timeout_link(conn->super.ctx->loop, &conn->super.ctx->http2.idle_timeout, &conn->_timeout_entry); } @@ -175,6 +174,9 @@ static int can_run_requests(h2o_http2_conn_t *conn) static void run_pending_requests(h2o_http2_conn_t *conn) { + if (h2o_timeout_is_linked(&conn->dos_mitigation.process_delay)) + return; + while (!h2o_linklist_is_empty(&conn->_pending_reqs) && can_run_requests(conn)) { /* fetch and detach a pending stream */ h2o_http2_stream_t *stream = H2O_STRUCT_FROM_MEMBER(h2o_http2_stream_t, _refs.link, conn->_pending_reqs.next); @@ -226,6 +228,16 @@ void h2o_http2_conn_unregister_stream(h2o_http2_conn_t *conn, h2o_http2_stream_t assert(h2o_http2_scheduler_is_open(&stream->_refs.scheduler)); h2o_http2_scheduler_close(&stream->_refs.scheduler); + /* Decrement reset_budget if the stream was reset by peer, otherwise increment. By doing so, we penalize connections that + * generate resets for >50% of requests. */ + if (stream->reset_by_peer) { + if (conn->dos_mitigation.reset_budget > 0) + --conn->dos_mitigation.reset_budget; + } else { + if (conn->dos_mitigation.reset_budget < conn->super.ctx->globalconf->http2.max_concurrent_requests_per_connection) + ++conn->dos_mitigation.reset_budget; + } + switch (stream->state) { case H2O_HTTP2_STREAM_STATE_IDLE: case H2O_HTTP2_STREAM_STATE_RECV_HEADERS: @@ -272,6 +284,8 @@ void close_connection_now(h2o_http2_conn_t *conn) h2o_hpack_dispose_header_table(&conn->_output_header_table); assert(h2o_linklist_is_empty(&conn->_pending_reqs)); h2o_timeout_unlink(&conn->_timeout_entry); + if (h2o_timeout_is_linked(&conn->dos_mitigation.process_delay)) + h2o_timeout_unlink(&conn->dos_mitigation.process_delay); h2o_buffer_dispose(&conn->_write.buf); if (conn->_write.buf_in_flight != NULL) h2o_buffer_dispose(&conn->_write.buf_in_flight); @@ -797,11 +811,19 @@ static int handle_rst_stream_frame(h2o_http2_conn_t *conn, h2o_http2_frame_t *fr return H2O_HTTP2_ERROR_PROTOCOL; } - stream = h2o_http2_conn_get_stream(conn, frame->stream_id); - if (stream != NULL) { + if ((stream = h2o_http2_conn_get_stream(conn, frame->stream_id)) == NULL) + return 0; + /* reset the stream */ + stream->reset_by_peer = 1; h2o_http2_stream_reset(conn, stream); - } + + /* setup process delay if we've just ran out of reset budget */ + if (conn->dos_mitigation.reset_budget == 0 && conn->super.ctx->globalconf->http2.dos_delay != 0 && + !h2o_timeout_is_linked(&conn->dos_mitigation.process_delay)) + h2o_timeout_link(conn->super.ctx->loop, &conn->super.ctx->http2.dos_delay_timeout, + &conn->dos_mitigation.process_delay); + /* TODO log */ return 0; @@ -1204,6 +1226,14 @@ static h2o_iovec_t log_priority_actual_weight(h2o_req_t *req) return h2o_iovec_init(s, len); } +static void on_dos_process_delay(h2o_timeout_entry_t *timer) +{ + h2o_http2_conn_t *conn = H2O_STRUCT_FROM_MEMBER(h2o_http2_conn_t, dos_mitigation.process_delay, timer); + + assert(!h2o_timeout_is_linked(&conn->dos_mitigation.process_delay)); + run_pending_requests(conn); +} + static h2o_http2_conn_t *create_conn(h2o_context_t *ctx, h2o_hostconf_t **hosts, h2o_socket_t *sock, struct timeval connected_at) { static const h2o_conn_callbacks_t callbacks = { @@ -1240,6 +1270,9 @@ static h2o_http2_conn_t *create_conn(h2o_context_t *ctx, h2o_hostconf_t **hosts, conn->_write.timeout_entry.cb = emit_writereq; h2o_http2_window_init(&conn->_write.window, &conn->peer_settings); + conn->dos_mitigation.process_delay.cb = on_dos_process_delay; + conn->dos_mitigation.reset_budget = conn->super.ctx->globalconf->http2.max_concurrent_requests_per_connection; + return conn; }