mirror of https://github.com/go-gitea/gitea.git
Browse Source
This adds a middleware for overload protection that is intended to help protect against malicious scrapers. It does this via [`codel`](https://github.com/bohde/codel), which will perform the following: 1. Limit the number of in-flight requests to some user-defined max 2. When in-flight requests have reached their begin queuing requests. Logged-in requests having priority above logged-out requests 3. Once a request has been queued for too long, it has a probabilistic chance to be rejected based on how overloaded the entire system is. When a server experiences more traffic than it can handle, this keeps latency low for logged-in users and rejects just enough requests from logged-out users to not overload the service. --------- Co-authored-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: Giteabot <teabot@gitea.io>pull/34202/head
Rowan Bohde
8 months ago
committed by
GitHub
10 changed files with 301 additions and 2 deletions
@ -0,0 +1,145 @@
@@ -0,0 +1,145 @@
|
||||
// Copyright 2025 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package common |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"net/http" |
||||
"strings" |
||||
|
||||
user_model "code.gitea.io/gitea/models/user" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/templates" |
||||
"code.gitea.io/gitea/modules/web/middleware" |
||||
giteacontext "code.gitea.io/gitea/services/context" |
||||
|
||||
"github.com/bohde/codel" |
||||
"github.com/go-chi/chi/v5" |
||||
) |
||||
|
||||
const tplStatus503 templates.TplName = "status/503" |
||||
|
||||
type Priority int |
||||
|
||||
func (p Priority) String() string { |
||||
switch p { |
||||
case HighPriority: |
||||
return "high" |
||||
case DefaultPriority: |
||||
return "default" |
||||
case LowPriority: |
||||
return "low" |
||||
default: |
||||
return fmt.Sprintf("%d", p) |
||||
} |
||||
} |
||||
|
||||
const ( |
||||
LowPriority = Priority(-10) |
||||
DefaultPriority = Priority(0) |
||||
HighPriority = Priority(10) |
||||
) |
||||
|
||||
// QoS implements quality of service for requests, based upon whether
|
||||
// or not the user is logged in. All traffic may get dropped, and
|
||||
// anonymous users are deprioritized.
|
||||
func QoS() func(next http.Handler) http.Handler { |
||||
if !setting.Service.QoS.Enabled { |
||||
return nil |
||||
} |
||||
|
||||
maxOutstanding := setting.Service.QoS.MaxInFlightRequests |
||||
if maxOutstanding <= 0 { |
||||
maxOutstanding = 10 |
||||
} |
||||
|
||||
c := codel.NewPriority(codel.Options{ |
||||
// The maximum number of waiting requests.
|
||||
MaxPending: setting.Service.QoS.MaxWaitingRequests, |
||||
// The maximum number of in-flight requests.
|
||||
MaxOutstanding: maxOutstanding, |
||||
// The target latency that a blocked request should wait
|
||||
// for. After this, it might be dropped.
|
||||
TargetLatency: setting.Service.QoS.TargetWaitTime, |
||||
}) |
||||
|
||||
return func(next http.Handler) http.Handler { |
||||
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { |
||||
ctx := req.Context() |
||||
|
||||
priority := requestPriority(ctx) |
||||
|
||||
// Check if the request can begin processing.
|
||||
err := c.Acquire(ctx, int(priority)) |
||||
if err != nil { |
||||
log.Error("QoS error, dropping request of priority %s: %v", priority, err) |
||||
renderServiceUnavailable(w, req) |
||||
return |
||||
} |
||||
|
||||
// Release long-polling immediately, so they don't always
|
||||
// take up an in-flight request
|
||||
if strings.Contains(req.URL.Path, "/user/events") { |
||||
c.Release() |
||||
} else { |
||||
defer c.Release() |
||||
} |
||||
|
||||
next.ServeHTTP(w, req) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
// requestPriority assigns a priority value for a request based upon
|
||||
// whether the user is logged in and how expensive the endpoint is
|
||||
func requestPriority(ctx context.Context) Priority { |
||||
// If the user is logged in, assign high priority.
|
||||
data := middleware.GetContextData(ctx) |
||||
if _, ok := data[middleware.ContextDataKeySignedUser].(*user_model.User); ok { |
||||
return HighPriority |
||||
} |
||||
|
||||
rctx := chi.RouteContext(ctx) |
||||
if rctx == nil { |
||||
return DefaultPriority |
||||
} |
||||
|
||||
// If we're operating in the context of a repo, assign low priority
|
||||
routePattern := rctx.RoutePattern() |
||||
if strings.HasPrefix(routePattern, "/{username}/{reponame}/") { |
||||
return LowPriority |
||||
} |
||||
|
||||
return DefaultPriority |
||||
} |
||||
|
||||
// renderServiceUnavailable will render an HTTP 503 Service
|
||||
// Unavailable page, providing HTML if the client accepts it.
|
||||
func renderServiceUnavailable(w http.ResponseWriter, req *http.Request) { |
||||
acceptsHTML := false |
||||
for _, part := range req.Header["Accept"] { |
||||
if strings.Contains(part, "text/html") { |
||||
acceptsHTML = true |
||||
break |
||||
} |
||||
} |
||||
|
||||
// If the client doesn't accept HTML, then render a plain text response
|
||||
if !acceptsHTML { |
||||
http.Error(w, "503 Service Unavailable", http.StatusServiceUnavailable) |
||||
return |
||||
} |
||||
|
||||
tmplCtx := giteacontext.TemplateContext{} |
||||
tmplCtx["Locale"] = middleware.Locale(w, req) |
||||
ctxData := middleware.GetContextData(req.Context()) |
||||
err := templates.HTMLRenderer().HTML(w, http.StatusServiceUnavailable, tplStatus503, ctxData, tmplCtx) |
||||
if err != nil { |
||||
log.Error("Error occurs again when rendering service unavailable page: %v", err) |
||||
w.WriteHeader(http.StatusInternalServerError) |
||||
_, _ = w.Write([]byte("Internal server error, please collect error logs and report to Gitea issue tracker")) |
||||
} |
||||
} |
||||
@ -0,0 +1,91 @@
@@ -0,0 +1,91 @@
|
||||
// Copyright 2025 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package common |
||||
|
||||
import ( |
||||
"net/http" |
||||
"testing" |
||||
|
||||
user_model "code.gitea.io/gitea/models/user" |
||||
"code.gitea.io/gitea/modules/web/middleware" |
||||
"code.gitea.io/gitea/services/contexttest" |
||||
|
||||
"github.com/go-chi/chi/v5" |
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestRequestPriority(t *testing.T) { |
||||
type test struct { |
||||
Name string |
||||
User *user_model.User |
||||
RoutePattern string |
||||
Expected Priority |
||||
} |
||||
|
||||
cases := []test{ |
||||
{ |
||||
Name: "Logged In", |
||||
User: &user_model.User{}, |
||||
Expected: HighPriority, |
||||
}, |
||||
{ |
||||
Name: "Sign In", |
||||
RoutePattern: "/user/login", |
||||
Expected: DefaultPriority, |
||||
}, |
||||
{ |
||||
Name: "Repo Home", |
||||
RoutePattern: "/{username}/{reponame}", |
||||
Expected: DefaultPriority, |
||||
}, |
||||
{ |
||||
Name: "User Repo", |
||||
RoutePattern: "/{username}/{reponame}/src/branch/main", |
||||
Expected: LowPriority, |
||||
}, |
||||
} |
||||
|
||||
for _, tc := range cases { |
||||
t.Run(tc.Name, func(t *testing.T) { |
||||
ctx, _ := contexttest.MockContext(t, "") |
||||
|
||||
if tc.User != nil { |
||||
data := middleware.GetContextData(ctx) |
||||
data[middleware.ContextDataKeySignedUser] = tc.User |
||||
} |
||||
|
||||
rctx := chi.RouteContext(ctx) |
||||
rctx.RoutePatterns = []string{tc.RoutePattern} |
||||
|
||||
assert.Exactly(t, tc.Expected, requestPriority(ctx)) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestRenderServiceUnavailable(t *testing.T) { |
||||
t.Run("HTML", func(t *testing.T) { |
||||
ctx, resp := contexttest.MockContext(t, "") |
||||
ctx.Req.Header.Set("Accept", "text/html") |
||||
|
||||
renderServiceUnavailable(resp, ctx.Req) |
||||
assert.Equal(t, http.StatusServiceUnavailable, resp.Code) |
||||
assert.Contains(t, resp.Header().Get("Content-Type"), "text/html") |
||||
|
||||
body := resp.Body.String() |
||||
assert.Contains(t, body, `lang="en-US"`) |
||||
assert.Contains(t, body, "503 Service Unavailable") |
||||
}) |
||||
|
||||
t.Run("plain", func(t *testing.T) { |
||||
ctx, resp := contexttest.MockContext(t, "") |
||||
ctx.Req.Header.Set("Accept", "text/plain") |
||||
|
||||
renderServiceUnavailable(resp, ctx.Req) |
||||
assert.Equal(t, http.StatusServiceUnavailable, resp.Code) |
||||
assert.Contains(t, resp.Header().Get("Content-Type"), "text/plain") |
||||
|
||||
body := resp.Body.String() |
||||
assert.Contains(t, body, "503 Service Unavailable") |
||||
}) |
||||
} |
||||
@ -0,0 +1,12 @@
@@ -0,0 +1,12 @@
|
||||
{{template "base/head" .}} |
||||
<div role="main" aria-label="503 Service Unavailable" class="page-content"> |
||||
<div class="ui container"> |
||||
<div class="status-page-error"> |
||||
<div class="status-page-error-title">503 Service Unavailable</div> |
||||
<div class="tw-text-center"> |
||||
<div class="tw-my-4">{{ctx.Locale.Tr "error503"}}</div> |
||||
</div> |
||||
</div> |
||||
</div> |
||||
</div> |
||||
{{template "base/footer" .}} |
||||
Loading…
Reference in new issue