| 1 | package main |
| 2 |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/aes" |
| 6 | "crypto/cipher" |
| 7 | "crypto/hmac" |
| 8 | "crypto/sha256" |
| 9 | "encoding/base64" |
| 10 | "encoding/hex" |
| 11 | "encoding/json" |
| 12 | "errors" |
| 13 | "fmt" |
| 14 | "io" |
| 15 | "log" |
| 16 | "net/http" |
| 17 | "strconv" |
| 18 | "strings" |
| 19 | "sync" |
| 20 | "time" |
| 21 |
| 22 | "github.com/jackc/pgx/v5" |
| 23 | "github.com/jackc/pgx/v5/pgxpool" |
| 24 | ) |
| 25 |
| 26 | // webhookPayload is what the API puts in delivery_job.payload. Body is frozen at fan-out so a retry sends the same bytes. |
| 27 | type webhookPayload struct { |
| 28 | Event string `json:"event"` |
| 29 | Body string `json:"body"` |
| 30 | } |
| 31 |
| 32 | // Stored response bodies are cut to this; the delivery log only needs a glance. |
| 33 | const maxResponseBody = 4 << 10 |
| 34 |
| 35 | type webhookSender struct { |
| 36 | db *pgxpool.Pool |
| 37 | client *http.Client |
| 38 | key []byte // AES-256 key for webhook_endpoint.secret, from WEBHOOK_SECRET_KEY |
| 39 | hosts *hostLimiter |
| 40 | } |
| 41 |
| 42 | // A job that finds its host busy waits this long; it is not the receiver's fault, so it records no attempt. |
| 43 | const hostBusyRetry = 5 * time.Second |
| 44 |
| 45 | func (w webhookSender) send(ctx context.Context, j Job) error { |
| 46 | if j.EndpointID == nil { |
| 47 | return permanentError{errors.New("webhook job has no endpoint_id")} |
| 48 | } |
| 49 | var p webhookPayload |
| 50 | if err := json.Unmarshal(j.Payload, &p); err != nil { |
| 51 | return permanentError{fmt.Errorf("decode webhook payload: %w", err)} |
| 52 | } |
| 53 |
| 54 | // Read at send time: the endpoint may have been disabled or its URL changed since enqueue. |
| 55 | var ( |
| 56 | url, sealed string |
| 57 | active bool |
| 58 | ) |
| 59 | err := w.db.QueryRow(ctx, |
| 60 | `SELECT url, secret, active FROM webhook_endpoint WHERE id = $1`, *j.EndpointID, |
| 61 | ).Scan(&url, &sealed, &active) |
| 62 | if errors.Is(err, pgx.ErrNoRows) { |
| 63 | return permanentError{errors.New("endpoint was deleted")} |
| 64 | } |
| 65 | if err != nil { |
| 66 | return err |
| 67 | } |
| 68 | if !active { |
| 69 | return permanentError{errors.New("endpoint is disabled")} |
| 70 | } |
| 71 | secret, err := openSecret(w.key, sealed) |
| 72 | if err != nil { |
| 73 | return err |
| 74 | } |
| 75 |
| 76 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(p.Body)) |
| 77 | if err != nil { |
| 78 | return permanentError{err} |
| 79 | } |
| 80 | for k, v := range signedHeaders(secret, j.ID, p.Event, []byte(p.Body), time.Now()) { |
| 81 | req.Header.Set(k, v) |
| 82 | } |
| 83 |
| 84 | if !w.hosts.acquire(req.URL.Host) { |
| 85 | return retryAfterError{fmt.Errorf("too many deliveries in flight to %s", req.URL.Host), hostBusyRetry} |
| 86 | } |
| 87 | defer w.hosts.release(req.URL.Host) |
| 88 |
| 89 | started := time.Now() |
| 90 | res, sendErr := w.client.Do(req) |
| 91 | var ( |
| 92 | status *int |
| 93 | resBody *string |
| 94 | ) |
| 95 | if sendErr == nil { |
| 96 | b, _ := io.ReadAll(io.LimitReader(res.Body, maxResponseBody)) |
| 97 | res.Body.Close() |
| 98 | status, resBody = &res.StatusCode, ptr(string(b)) |
| 99 | } |
| 100 | w.recordAttempt(ctx, j.ID, started, req.Header, status, resBody, sendErr) |
| 101 |
| 102 | if sendErr != nil { |
| 103 | return sendErr |
| 104 | } |
| 105 | return w.classify(ctx, *j.EndpointID, res) |
| 106 | } |
| 107 |
| 108 | func (w webhookSender) classify(ctx context.Context, endpointID string, res *http.Response) error { |
| 109 | switch code := res.StatusCode; { |
| 110 | case code >= 200 && code <= 299: |
| 111 | return nil |
| 112 | case code == http.StatusGone: |
| 113 | _, err := w.db.Exec(ctx, ` |
| 114 | UPDATE webhook_endpoint |
| 115 | SET active = false, disabled_reason = 'The endpoint responded 410 Gone.', updated_at = now() |
| 116 | WHERE id = $1`, endpointID) |
| 117 | if err != nil { |
| 118 | return err |
| 119 | } |
| 120 | return permanentError{errors.New("endpoint responded 410 Gone and was disabled")} |
| 121 | case code == http.StatusTooManyRequests: |
| 122 | err := fmt.Errorf("endpoint responded %d", code) |
| 123 | if after, ok := parseRetryAfter(res.Header.Get("Retry-After"), time.Now()); ok { |
| 124 | return retryAfterError{err, after} |
| 125 | } |
| 126 | return err |
| 127 | default: |
| 128 | // ponytail: every other status retries with backoff, 4xx included; a receiver fixing its config then catches up. Fail fast on 4xx if retries waste too much. |
| 129 | return fmt.Errorf("endpoint responded %d", code) |
| 130 | } |
| 131 | } |
| 132 |
| 133 | // A failure to write the attempt is logged, not returned: the job's own result matters more than its log line. |
| 134 | func (w webhookSender) recordAttempt(ctx context.Context, jobID string, started time.Time, headers http.Header, status *int, body *string, sendErr error) { |
| 135 | flat := make(map[string]string, len(headers)) |
| 136 | for k := range headers { |
| 137 | flat[k] = headers.Get(k) |
| 138 | } |
| 139 | var errText *string |
| 140 | if sendErr != nil { |
| 141 | errText = ptr(sendErr.Error()) |
| 142 | } |
| 143 | _, err := w.db.Exec(ctx, ` |
| 144 | INSERT INTO delivery_attempt (job_id, started_at, duration_ms, status_code, error, request_headers, response_body) |
| 145 | VALUES ($1, $2, $3, $4, $5, $6, $7)`, |
| 146 | jobID, started, time.Since(started).Milliseconds(), status, errText, flat, body) |
| 147 | if err != nil { |
| 148 | log.Printf("job %s: record attempt: %v", jobID, err) |
| 149 | } |
| 150 | } |
| 151 |
| 152 | // The timestamp is signed with the body so a receiver can reject replays; X-Hub-Signature-256 signs the body alone for GitHub-style receivers. |
| 153 | func signedHeaders(secret []byte, deliveryID, event string, body []byte, now time.Time) map[string]string { |
| 154 | ts := strconv.FormatInt(now.Unix(), 10) |
| 155 | return map[string]string{ |
| 156 | "Content-Type": "application/json", |
| 157 | "User-Agent": "Ghost-Hookshot/1", |
| 158 | "X-Ghost-Event": event, |
| 159 | "X-Ghost-Delivery": deliveryID, |
| 160 | "X-Ghost-Timestamp": ts, |
| 161 | "X-Ghost-Signature-256": "sha256=" + hmacHex(secret, []byte(ts+"."), body), |
| 162 | "X-Hub-Signature-256": "sha256=" + hmacHex(secret, body), |
| 163 | } |
| 164 | } |
| 165 |
| 166 | func hmacHex(secret []byte, parts ...[]byte) string { |
| 167 | mac := hmac.New(sha256.New, secret) |
| 168 | for _, p := range parts { |
| 169 | mac.Write(p) |
| 170 | } |
| 171 | return hex.EncodeToString(mac.Sum(nil)) |
| 172 | } |
| 173 |
| 174 | // openSecret reads the format sealWebhookSecret in apps/api writes: "v1:" + base64(12-byte nonce || AES-256-GCM ciphertext || tag). |
| 175 | func openSecret(key []byte, sealed string) ([]byte, error) { |
| 176 | if len(key) == 0 { |
| 177 | return nil, errors.New("WEBHOOK_SECRET_KEY is not set") |
| 178 | } |
| 179 | raw, ok := strings.CutPrefix(sealed, "v1:") |
| 180 | if !ok { |
| 181 | return nil, errors.New("webhook secret has an unknown format") |
| 182 | } |
| 183 | data, err := base64.StdEncoding.DecodeString(raw) |
| 184 | if err != nil { |
| 185 | return nil, fmt.Errorf("webhook secret: %w", err) |
| 186 | } |
| 187 | block, err := aes.NewCipher(key) |
| 188 | if err != nil { |
| 189 | return nil, err |
| 190 | } |
| 191 | gcm, err := cipher.NewGCM(block) |
| 192 | if err != nil { |
| 193 | return nil, err |
| 194 | } |
| 195 | if len(data) < gcm.NonceSize() { |
| 196 | return nil, errors.New("webhook secret is too short") |
| 197 | } |
| 198 | nonce, ciphertext := data[:gcm.NonceSize()], data[gcm.NonceSize():] |
| 199 | plain, err := gcm.Open(nil, nonce, ciphertext, nil) |
| 200 | if err != nil { |
| 201 | return nil, fmt.Errorf("webhook secret: %w", err) |
| 202 | } |
| 203 | return plain, nil |
| 204 | } |
| 205 |
| 206 | func parseRetryAfter(v string, now time.Time) (time.Duration, bool) { |
| 207 | if v == "" { |
| 208 | return 0, false |
| 209 | } |
| 210 | if s, err := strconv.Atoi(v); err == nil && s >= 0 { |
| 211 | return time.Duration(s) * time.Second, true |
| 212 | } |
| 213 | if t, err := http.ParseTime(v); err == nil { |
| 214 | return max(t.Sub(now), 0), true |
| 215 | } |
| 216 | return 0, false |
| 217 | } |
| 218 |
| 219 | func ptr[T any](v T) *T { return &v } |
| 220 |
| 221 | // hostLimiter caps sends in flight per host, so one slow receiver cannot hold every worker. A nil limiter allows everything. |
| 222 | // ponytail: per process, so with several delivery instances the real cap is instances x max; move it to Postgres if that matters. |
| 223 | type hostLimiter struct { |
| 224 | mu sync.Mutex |
| 225 | max int |
| 226 | inUse map[string]int |
| 227 | } |
| 228 |
| 229 | func newHostLimiter(max int) *hostLimiter { |
| 230 | return &hostLimiter{max: max, inUse: map[string]int{}} |
| 231 | } |
| 232 |
| 233 | func (h *hostLimiter) acquire(host string) bool { |
| 234 | if h == nil { |
| 235 | return true |
| 236 | } |
| 237 | h.mu.Lock() |
| 238 | defer h.mu.Unlock() |
| 239 | if h.inUse[host] >= h.max { |
| 240 | return false |
| 241 | } |
| 242 | h.inUse[host]++ |
| 243 | return true |
| 244 | } |
| 245 |
| 246 | func (h *hostLimiter) release(host string) { |
| 247 | if h == nil { |
| 248 | return |
| 249 | } |
| 250 | h.mu.Lock() |
| 251 | defer h.mu.Unlock() |
| 252 | if h.inUse[host]--; h.inUse[host] <= 0 { |
| 253 | delete(h.inUse, host) |
| 254 | } |
| 255 | } |