Idempotency & Safe Retries

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

// NAIVE ANTI-PATTERN: Non-atomic idempotency check
async function handlePayment(req: Request): Promise<Response> {
  const key = req.headers['idempotency-key'];

  // 1. Check if request was already processed
  const existing = await db.query('SELECT * FROM payments WHERE idempotency_key = $1', [key]);
  if (existing.rows.length > 0) {
    return Response.json(existing.rows[0].response);
  }

  // RACE CONDITION WINDOW:
  // If a user double-clicks "Submit", or two retries arrive within 5ms of each other,
  // BOTH threads pass the SELECT check above simultaneously!

  // 2. Both threads charge the card via Stripe!
  const charge = await stripe.charges.create({ amount: req.body.amount });

  // 3. Both threads insert into database
  await db.query('INSERT INTO payments (idempotency_key, charge_id) VALUES ($1, $2)', [key, charge.id]);

  return Response.json(charge);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148

// TEACHING SKETCH: complete the lease and ownership rules before production use
import { createHash } from 'crypto';

export type IdempotencyStatus = 'PROCESSING' | 'SUCCEEDED' | 'FAILED';

export interface IdempotencyRecord {
  key: string;
  payloadHash: string; // SHA-256 hash of request body
  status: IdempotencyStatus;
  statusCode: number;
  responseBody: string;
  createdAt: Date;
}

export class StripeGradeIdempotencyEngine {
  private db: any;       // PostgreSQL client
  private redis: any;    // Redis client for reservation locks

  constructor(db: any, redis: any) {
    this.db = db;
    this.redis = redis;
  }

  public async processRequest(
    key: string,
    requestPayload: any,
    businessHandler: () => Promise<{ statusCode: number; body: any }>
  ): Promise<{ statusCode: number; body: any }> {
    // 1. Compute cryptographic hash of request parameters
    const currentHash = createHash('sha256')
      .update(JSON.stringify(requestPayload))
      .digest('hex');

    // 2. Acquire short-lived reservation lock in Redis (e.g. 30s TTL)
    const lockAcquired = await this.redis.set(`lock:idemp:${key}`, '1', 'NX', 'EX', 30);
    if (!lockAcquired) {
      // Another concurrent thread is actively processing this exact key!
      // Return HTTP 409 Conflict with standard Retry-After header
      return {
        statusCode: 409,
        body: { error: 'Concurrent request in flight with the same idempotency key. Please retry.' },
      };
    }

    try {
      // 3. Inspect persistent idempotency table
      const existing = await this.db.query(
        'SELECT * FROM idempotency_keys WHERE key = $1',
        [key]
      );

      if (existing.rows.length > 0) {
        const record: IdempotencyRecord = existing.rows[0];

        // INVARIANT: Verify request payload matches the original submission!
        if (record.payloadHash !== currentHash) {
          return {
            statusCode: 422,
            body: {
              error: 'Unprocessable Entity: Idempotency key reused with mismatched request parameters!',
            },
          };
        }

        // If previous request succeeded, replay the exact cached response!
        if (record.status === 'SUCCEEDED') {
          console.log(`[Idempotency] Replaying cached response for key: ${key}`);
          return {
            statusCode: record.statusCode,
            body: JSON.parse(record.responseBody),
          };
        }

        // CRITICAL: Prevent duplicate execution if request is already in-flight!
        if (record.status === 'PROCESSING') {
          return {
            statusCode: 409,
            body: {
              error: 'Conflict: An operation with this idempotency key is actively being processed. Please retry with backoff.',
            },
          };
        }

        // If previously marked FAILED, allow re-execution by transitioning back to PROCESSING
        if (record.status === 'FAILED') {
          await this.db.query(
            `UPDATE idempotency_keys
             SET status = 'PROCESSING',
                 payload_hash = $1
             WHERE key = $2`,
            [currentHash, key]
          );
        }
      } else {
        // 4. Insert initial row with status 'PROCESSING'
        await this.db.query(
          `INSERT INTO idempotency_keys (key, payload_hash, status, created_at)
           VALUES ($1, $2, 'PROCESSING', NOW())`,
          [key, currentHash]
        );
      }

      // 5. Execute core business logic (e.g. charge payment) inside try-catch to record failures
      let result: { statusCode: number; body: any };
      try {
        result = await businessHandler();
      } catch (err: any) {
        // Record FAILED status so subsequent retries are unblocked and can re-attempt
        await this.db.query(
          `UPDATE idempotency_keys
           SET status = 'FAILED',
               status_code = 500,
               response_body = $1
           WHERE key = $2`,
          [JSON.stringify({ error: err?.message || 'Internal processing error' }), key]
        );
        throw err;
      }

      // 6. Atomically update record to 'SUCCEEDED' with cached response
      await this.db.query(
        `UPDATE idempotency_keys
         SET status = 'SUCCEEDED',
             status_code = $1,
             response_body = $2
         WHERE key = $3`,
        [result.statusCode, JSON.stringify(result.body), key]
      );

      return result;
    } finally {
      // 7. Always release Redis reservation lock
      await this.redis.del(`lock:idemp:${key}`);
    }
  }
}

// FULL-JITTER EXPONENTIAL BACKOFF
export function calculateFullJitterSleepMs(
  attempt: number,
  baseDelayMs: number = 100,
  maxDelayMs: number = 10000
): number {
  // Exponential backoff ceiling: base * 2^attempt
  const exponentialCap = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attempt));
  // Full Jitter: Pick random uniform value between 0 and exponentialCap
  return Math.floor(Math.random() * exponentialCap);
}