Part 7: Testing, Performance, and Production Deployment
The Database Index Disaster
October 2022. We launched our new gRPC order service to production. Initial load tests showed great results: 2,000 requests/second, 50ms average latency. We celebrated.
Day 3 of production: The service started timing out. Latency spiked to 15 seconds. Orders were failing. Database CPU hit 100%. After 4 hours of desperate debugging, we found the issue: no database indexes on the orders table. Our test data had 1,000 orders. Production had 500,000 orders.
That oversight cost approximately $80,000 in lost revenue, emergency infrastructure scaling, and remediation work. This part covers everything about testing, performance optimization, and production deployment—lessons learned from managing 23 gRPC services in production.
# Install k6
brew install k6
# Run load test
k6 run scripts/load-test.js
# Run with specific parameters
k6 run --vus 50 --duration 5m scripts/load-test.js
# Run and export results
k6 run --out json=results.json scripts/load-test.js
-- Add indexes (learned the hard way!)
CREATE INDEX idx_orders_user_id ON orders(user_id);
CREATE INDEX idx_orders_status ON orders(status);
CREATE INDEX idx_orders_created_at ON orders(created_at DESC);
CREATE INDEX idx_orders_user_status ON orders(user_id, status);
-- Composite index for common queries
CREATE INDEX idx_orders_user_created ON orders(user_id, created_at DESC);
-- Order items
CREATE INDEX idx_order_items_order_id ON order_items(order_id);
CREATE INDEX idx_order_items_product_id ON order_items(product_id);
-- Analyze query performance
EXPLAIN ANALYZE
SELECT * FROM orders
WHERE user_id = 'usr_123'
AND status = 1
ORDER BY created_at DESC
LIMIT 10;
// src/repositories/product.repository.ts
export class ProductRepository {
private readonly CACHE_TTL = 3600; // 1 hour
private readonly CACHE_PREFIX = 'product:';
async findById(id: string): Promise<Product | null> {
// 1. Try cache first
const cached = await redis.get<Product>(`${this.CACHE_PREFIX}${id}`);
if (cached) {
return cached;
}
// 2. Query database
const product = await database.query<Product>(
'SELECT * FROM products WHERE id = $1',
[id]
);
// 3. Cache result
if (product[0]) {
await redis.set(
`${this.CACHE_PREFIX}${id}`,
product[0],
this.CACHE_TTL
);
}
return product[0] || null;
}
async batchFindByIds(ids: string[]): Promise<Map<string, Product>> {
// Multi-get from cache
const cacheKeys = ids.map(id => `${this.CACHE_PREFIX}${id}`);
const cached = await redis.mget(cacheKeys);
const result = new Map<string, Product>();
const missingIds: string[] = [];
ids.forEach((id, index) => {
if (cached[index]) {
result.set(id, cached[index]);
} else {
missingIds.push(id);
}
});
// Fetch missing from database
if (missingIds.length > 0) {
const products = await database.query<Product>(
`SELECT * FROM products WHERE id = ANY($1)`,
[missingIds]
);
// Cache and add to result
for (const product of products) {
result.set(product.id, product);
await redis.set(
`${this.CACHE_PREFIX}${product.id}`,
product,
this.CACHE_TTL
);
}
}
return result;
}
}
# Dockerfile
FROM node:20-alpine AS builder
WORKDIR /app
# Copy package files
COPY package*.json ./
COPY tsconfig.json ./
# Install dependencies
RUN npm ci
# Copy source
COPY protos ./protos
COPY src ./src
# Generate proto code and build
RUN npm run proto:generate
RUN npm run build
# Production image
FROM node:20-alpine
WORKDIR /app
# Install production dependencies
COPY package*.json ./
RUN npm ci --only=production && npm cache clean --force
# Copy built app
COPY --from=builder /app/dist ./dist
COPY --from=builder /app/protos ./protos
# Create non-root user
RUN addgroup -g 1001 -S nodejs && \
adduser -S nodejs -u 1001
USER nodejs
EXPOSE 50051
CMD ["node", "dist/server.js"]
