Part 5: Error Handling and Interceptors in gRPC

The 3 AM Production Incident

It was 3:17 AM when I got the PagerDuty alert. Our payment processing gRPC service was returning cryptic errors: "code: 2, message: unknown error". No stack traces. No context. No idea what was broken. The mobile app showed "Something went wrong" to thousands of users trying to complete purchases.

After 2 hours of digging through logs, I found the issue: a database connection timeout. But the error handling was so poor that the real error was swallowed, making debugging nearly impossible. That night cost us approximately $45,000 in lost revenue and emergency response.

This part covers everything I learned about proper error handling in gRPC—the hard way.

gRPC Status Codes

Standard Status Codes

// src/utils/grpc-status.ts
import * as grpc from '@grpc/grpc-js';

export const GrpcStatus = {
  // Success
  OK: grpc.status.OK, // 0

  // Client errors
  CANCELLED: grpc.status.CANCELLED, // 1 - Request cancelled
  INVALID_ARGUMENT: grpc.status.INVALID_ARGUMENT, // 3 - Invalid request
  NOT_FOUND: grpc.status.NOT_FOUND, // 5 - Resource not found
  ALREADY_EXISTS: grpc.status.ALREADY_EXISTS, // 6 - Resource exists
  PERMISSION_DENIED: grpc.status.PERMISSION_DENIED, // 7 - No permission
  UNAUTHENTICATED: grpc.status.UNAUTHENTICATED, // 16 - Not authenticated
  RESOURCE_EXHAUSTED: grpc.status.RESOURCE_EXHAUSTED, // 8 - Rate limit/quota

  // Server errors
  FAILED_PRECONDITION: grpc.status.FAILED_PRECONDITION, // 9 - State issue
  ABORTED: grpc.status.ABORTED, // 10 - Concurrency conflict
  OUT_OF_RANGE: grpc.status.OUT_OF_RANGE, // 11 - Invalid range
  UNIMPLEMENTED: grpc.status.UNIMPLEMENTED, // 12 - Not implemented
  INTERNAL: grpc.status.INTERNAL, // 13 - Server error
  UNAVAILABLE: grpc.status.UNAVAILABLE, // 14 - Service unavailable
  DATA_LOSS: grpc.status.DATA_LOSS, // 15 - Data loss/corruption
  DEADLINE_EXCEEDED: grpc.status.DEADLINE_EXCEEDED, // 4 - Timeout
  UNKNOWN: grpc.status.UNKNOWN, // 2 - Unknown error
};

// Human-readable descriptions
export const StatusDescriptions: Record<number, string> = {
  [grpc.status.OK]: 'Success',
  [grpc.status.CANCELLED]: 'Request was cancelled',
  [grpc.status.INVALID_ARGUMENT]: 'Invalid request parameters',
  [grpc.status.DEADLINE_EXCEEDED]: 'Request timeout exceeded',
  [grpc.status.NOT_FOUND]: 'Resource not found',
  [grpc.status.ALREADY_EXISTS]: 'Resource already exists',
  [grpc.status.PERMISSION_DENIED]: 'Permission denied',
  [grpc.status.RESOURCE_EXHAUSTED]: 'Resource exhausted (rate limit or quota)',
  [grpc.status.FAILED_PRECONDITION]: 'Operation rejected due to system state',
  [grpc.status.ABORTED]: 'Operation aborted (typically due to concurrency issue)',
  [grpc.status.OUT_OF_RANGE]: 'Operation attempted past valid range',
  [grpc.status.UNIMPLEMENTED]: 'Operation not implemented',
  [grpc.status.INTERNAL]: 'Internal server error',
  [grpc.status.UNAVAILABLE]: 'Service unavailable',
  [grpc.status.DATA_LOSS]: 'Unrecoverable data loss or corruption',
  [grpc.status.UNAUTHENTICATED]: 'Authentication required',
  [grpc.status.UNKNOWN]: 'Unknown error',
};

Custom Error Classes

// src/utils/errors.ts
import * as grpc from '@grpc/grpc-js';

export class GrpcError extends Error {
  constructor(
    public code: grpc.status,
    message: string,
    public details?: any
  ) {
    super(message);
    this.name = 'GrpcError';
    Object.setPrototypeOf(this, GrpcError.prototype);
  }

  toServiceError(): grpc.ServiceError {
    return {
      name: this.name,
      message: this.message,
      code: this.code,
      details: this.details,
    };
  }
}

// Specific error types for common scenarios
export class NotFoundError extends GrpcError {
  constructor(resource: string, id: string) {
    super(
      grpc.status.NOT_FOUND,
      `${resource} with ID "${id}" not found`,
      { resource, id }
    );
    this.name = 'NotFoundError';
  }
}

export class ValidationError extends GrpcError {
  constructor(message: string, fieldErrors: Record<string, string>) {
    super(grpc.status.INVALID_ARGUMENT, message, { fieldErrors });
    this.name = 'ValidationError';
  }
}

export class UnauthorizedError extends GrpcError {
  constructor(message: string = 'Authentication required') {
    super(grpc.status.UNAUTHENTICATED, message);
    this.name = 'UnauthorizedError';
  }
}

export class ForbiddenError extends GrpcError {
  constructor(message: string = 'Permission denied') {
    super(grpc.status.PERMISSION_DENIED, message);
    this.name = 'ForbiddenError';
  }
}

export class ConflictError extends GrpcError {
  constructor(message: string, details?: any) {
    super(grpc.status.ALREADY_EXISTS, message, details);
    this.name = 'ConflictError';
  }
}

export class RateLimitError extends GrpcError {
  constructor(retryAfter?: number) {
    super(
      grpc.status.RESOURCE_EXHAUSTED,
      'Rate limit exceeded',
      { retryAfter }
    );
    this.name = 'RateLimitError';
  }
}

export class PreconditionError extends GrpcError {
  constructor(message: string, details?: any) {
    super(grpc.status.FAILED_PRECONDITION, message, details);
    this.name = 'PreconditionError';
  }
}

export class InternalError extends GrpcError {
  constructor(message: string = 'Internal server error', details?: any) {
    super(grpc.status.INTERNAL, message, details);
    this.name = 'InternalError';
  }
}

export class UnavailableError extends GrpcError {
  constructor(message: string = 'Service temporarily unavailable') {
    super(grpc.status.UNAVAILABLE, message);
    this.name = 'UnavailableError';
  }
}

export class DeadlineExceededError extends GrpcError {
  constructor(operation: string) {
    super(
      grpc.status.DEADLINE_EXCEEDED,
      `Operation "${operation}" exceeded deadline`,
      { operation }
    );
    this.name = 'DeadlineExceededError';
  }
}

Error Handling Interceptor

// src/grpc/interceptors/error.interceptor.ts
import * as grpc from '@grpc/grpc-js';
import { GrpcError, InternalError } from '@utils/errors';
import { logger } from '@config/logger';

export function errorInterceptor(
  call: grpc.ServerUnaryCall<any, any> | grpc.ServerWritableStream<any, any>,
  callback: grpc.sendUnaryData<any> | undefined,
  next: () => void
): void {
  if (!callback) {
    // For streaming calls, wrap the call object
    return next();
  }

  const originalCallback = callback;
  const method = call.getPath();
  const requestId = generateRequestId();

  // Wrap callback to catch errors
  callback = (error: any, response?: any) => {
    if (!error) {
      return originalCallback(null, response);
    }

    // Handle GrpcError instances
    if (error instanceof GrpcError) {
      logger.warn('gRPC business error', {
        requestId,
        method,
        errorName: error.name,
        errorMessage: error.message,
        errorCode: error.code,
        errorDetails: error.details,
        userId: call.authContext?.userId,
      });

      return originalCallback(error.toServiceError());
    }

    // Handle database errors
    if (error.code === 'ECONNREFUSED' || error.code === 'ETIMEDOUT') {
      logger.error('Database connection error', {
        requestId,
        method,
        error: error.message,
        userId: call.authContext?.userId,
      });

      return originalCallback(
        new UnavailableError('Database temporarily unavailable').toServiceError()
      );
    }

    // Handle validation errors (from Zod or similar)
    if (error.name === 'ZodError') {
      logger.warn('Validation error', {
        requestId,
        method,
        errors: error.errors,
        userId: call.authContext?.userId,
      });

      const fieldErrors = error.errors.reduce((acc: any, err: any) => {
        acc[err.path.join('.')] = err.message;
        return acc;
      }, {});

      return originalCallback(
        new ValidationError('Validation failed', fieldErrors).toServiceError()
      );
    }

    // Handle native gRPC errors
    if (error.code && typeof error.code === 'number') {
      logger.error('gRPC error', {
        requestId,
        method,
        code: error.code,
        message: error.message,
        userId: call.authContext?.userId,
      });

      return originalCallback(error);
    }

    // Unknown errors - never expose internal details to client
    logger.error('Unexpected error', {
      requestId,
      method,
      error: error.message,
      stack: error.stack,
      userId: call.authContext?.userId,
    });

    // Return generic error to client
    return originalCallback(
      new InternalError('An unexpected error occurred', {
        requestId, // Include request ID for support
      }).toServiceError()
    );
  };

  next();
}

function generateRequestId(): string {
  return `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}

Input Validation with Zod

// src/utils/validation.ts
import { z } from 'zod';
import { ValidationError } from './errors';

// Common validation schemas
export const validators = {
  // ID validation
  id: z.string().min(1, 'ID is required'),
  
  // Email validation
  email: z.string().email('Invalid email address'),
  
  // Pagination
  pageSize: z.number().int().min(1).max(100),
  pageToken: z.string().optional(),
  
  // Money validation
  money: z.object({
    currencyCode: z.string().length(3, 'Currency code must be 3 characters'),
    units: z.bigint(),
    nanos: z.number().int().min(0).max(999_999_999),
  }),
  
  // Address validation
  address: z.object({
    street: z.string().min(1, 'Street is required'),
    city: z.string().min(1, 'City is required'),
    state: z.string().min(2, 'State must be at least 2 characters'),
    postalCode: z.string().min(3, 'Postal code is required'),
    country: z.string().length(2, 'Country must be 2-letter code'),
  }),
};

// Order validation schemas
export const orderValidation = {
  createOrder: z.object({
    idempotencyKey: z
      .string()
      .min(1, 'Idempotency key is required')
      .max(128, 'Idempotency key too long'),
    userId: validators.id,
    items: z
      .array(
        z.object({
          productId: validators.id,
          quantity: z.number().int().min(1).max(100),
        })
      )
      .min(1, 'At least one item is required')
      .max(50, 'Maximum 50 items per order'),
    shippingAddress: validators.address,
  }),

  getOrder: z.object({
    id: validators.id,
  }),

  listOrders: z.object({
    userId: validators.id.optional(),
    status: z.number().int().min(0).max(7).optional(),
    pageInfo: z
      .object({
        pageSize: validators.pageSize.default(10),
        pageToken: validators.pageToken,
      })
      .optional(),
  }),

  updateOrderStatus: z.object({
    id: validators.id,
    status: z.number().int().min(1).max(7),
  }),

  cancelOrder: z.object({
    id: validators.id,
    reason: z.string().max(500).optional(),
  }),
};

// Validation helper
export function validate<T>(schema: z.ZodSchema<T>, data: unknown): T {
  try {
    return schema.parse(data);
  } catch (error) {
    if (error instanceof z.ZodError) {
      const fieldErrors = error.errors.reduce((acc, err) => {
        acc[err.path.join('.')] = err.message;
        return acc;
      }, {} as Record<string, string>);

      throw new ValidationError('Validation failed', fieldErrors);
    }
    throw error;
  }
}

Using Validation in Handlers

// src/grpc/handlers/order.handler.ts
import { validate, orderValidation } from '@utils/validation';
import { NotFoundError, PreconditionError } from '@utils/errors';

export class OrderHandler {
  async createOrder(
    call: grpc.ServerUnaryCall<CreateOrderRequest, Order>,
    callback: grpc.sendUnaryData<Order>
  ): Promise<void> {
    try {
      // Validate request
      const validatedRequest = validate(
        orderValidation.createOrder,
        call.request
      );

      // Create order
      const order = await this.orderService.createOrder(validatedRequest);

      callback(null, order);
    } catch (error: any) {
      callback(error);
    }
  }

  async getOrder(
    call: grpc.ServerUnaryCall<GetOrderRequest, Order>,
    callback: grpc.sendUnaryData<Order>
  ): Promise<void> {
    try {
      // Validate request
      const { id } = validate(orderValidation.getOrder, call.request);

      // Get order
      const order = await this.orderService.getOrder(id, call.authContext!);

      callback(null, order);
    } catch (error: any) {
      callback(error);
    }
  }

  async updateOrderStatus(
    call: grpc.ServerUnaryCall<UpdateOrderStatusRequest, Order>,
    callback: grpc.sendUnaryData<Order>
  ): Promise<void> {
    try {
      // Validate request
      const { id, status } = validate(
        orderValidation.updateOrderStatus,
        call.request
      );

      // Business validation
      const currentOrder = await this.orderService.getOrder(
        id,
        call.authContext!
      );

      if (![1, 2].includes(currentOrder.status)) {
        throw new PreconditionError(
          'Order cannot be updated in its current state',
          { currentStatus: currentOrder.status, requestedStatus: status }
        );
      }

      // Update order
      const order = await this.orderService.updateOrderStatus(
        id,
        status,
        call.authContext!
      );

      callback(null, order);
    } catch (error: any) {
      callback(error);
    }
  }
}

Advanced Interceptors

1. Logging Interceptor

// src/grpc/interceptors/logging.interceptor.ts
import * as grpc from '@grpc/grpc-js';
import { logger } from '@config/logger';

export function loggingInterceptor(
  call: grpc.ServerUnaryCall<any, any> | grpc.ServerWritableStream<any, any>,
  callback: grpc.sendUnaryData<any> | undefined,
  next: () => void
): void {
  const method = call.getPath();
  const startTime = Date.now();
  const requestId = `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;

  logger.info('gRPC request started', {
    requestId,
    method,
    userId: call.authContext?.userId,
  });

  if (!callback) {
    // Streaming call
    const stream = call as grpc.ServerWritableStream<any, any>;

    stream.on('error', error => {
      const duration = Date.now() - startTime;
      logger.error('gRPC stream error', {
        requestId,
        method,
        duration,
        error: error.message,
      });
    });

    stream.on('finish', () => {
      const duration = Date.now() - startTime;
      logger.info('gRPC stream finished', {
        requestId,
        method,
        duration,
      });
    });

    return next();
  }

  const originalCallback = callback;

  callback = (error, response) => {
    const duration = Date.now() - startTime;

    if (error) {
      logger.error('gRPC request failed', {
        requestId,
        method,
        duration,
        errorCode: error.code,
        errorMessage: error.message,
        userId: call.authContext?.userId,
      });
    } else {
      logger.info('gRPC request completed', {
        requestId,
        method,
        duration,
        userId: call.authContext?.userId,
      });
    }

    originalCallback(error, response);
  };

  next();
}

2. Metrics Interceptor

// src/grpc/interceptors/metrics.interceptor.ts
import * as grpc from '@grpc/grpc-js';
import { Counter, Histogram } from 'prom-client';

// Prometheus metrics
const requestCounter = new Counter({
  name: 'grpc_requests_total',
  help: 'Total number of gRPC requests',
  labelNames: ['method', 'status'],
});

const requestDuration = new Histogram({
  name: 'grpc_request_duration_seconds',
  help: 'Duration of gRPC requests in seconds',
  labelNames: ['method', 'status'],
  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
});

const activeRequests = new Counter({
  name: 'grpc_active_requests',
  help: 'Number of active gRPC requests',
  labelNames: ['method'],
});

export function metricsInterceptor(
  call: grpc.ServerUnaryCall<any, any> | grpc.ServerWritableStream<any, any>,
  callback: grpc.sendUnaryData<any> | undefined,
  next: () => void
): void {
  const method = call.getPath();
  const startTime = Date.now();

  // Increment active requests
  activeRequests.inc({ method });

  if (!callback) {
    // Streaming call - simplified metrics
    return next();
  }

  const originalCallback = callback;

  callback = (error, response) => {
    const duration = (Date.now() - startTime) / 1000;
    const status = error ? error.code : grpc.status.OK;

    // Record metrics
    requestCounter.inc({ method, status });
    requestDuration.observe({ method, status }, duration);
    activeRequests.dec({ method });

    originalCallback(error, response);
  };

  next();
}

3. Timeout Interceptor

// src/grpc/interceptors/timeout.interceptor.ts
import * as grpc from '@grpc/grpc-js';
import { DeadlineExceededError } from '@utils/errors';
import { logger } from '@config/logger';

const DEFAULT_TIMEOUT_MS = 30000; // 30 seconds

const methodTimeouts: Record<string, number> = {
  '/order.v1.OrderService/CreateOrder': 10000, // 10 seconds
  '/order.v1.OrderService/GetOrder': 5000, // 5 seconds
  '/product.v1.ProductService/SearchProducts': 15000, // 15 seconds
};

export function timeoutInterceptor(
  call: grpc.ServerUnaryCall<any, any> | grpc.ServerWritableStream<any, any>,
  callback: grpc.sendUnaryData<any> | undefined,
  next: () => void
): void {
  if (!callback) {
    return next(); // Skip for streaming calls
  }

  const method = call.getPath();
  const timeout = methodTimeouts[method] || DEFAULT_TIMEOUT_MS;

  let timeoutHandle: NodeJS.Timeout;
  let completed = false;

  const originalCallback = callback;

  // Set timeout
  timeoutHandle = setTimeout(() => {
    if (!completed) {
      completed = true;

      logger.warn('Request timeout exceeded', { method, timeout });

      originalCallback(
        new DeadlineExceededError(method).toServiceError()
      );
    }
  }, timeout);

  // Wrap callback to clear timeout
  callback = (error, response) => {
    if (!completed) {
      completed = true;
      clearTimeout(timeoutHandle);
      originalCallback(error, response);
    }
  };

  next();
}

4. Request Size Limit Interceptor

// src/grpc/interceptors/size-limit.interceptor.ts
import * as grpc from '@grpc/grpc-js';
import { GrpcError } from '@utils/errors';
import { logger } from '@config/logger';

const MAX_REQUEST_SIZE = 1024 * 1024; // 1MB default

const methodSizeLimits: Record<string, number> = {
  '/order.v1.OrderService/CreateOrder': 512 * 1024, // 512KB
  '/product.v1.ProductService/UpdateProduct': 2 * 1024 * 1024, // 2MB
};

export function sizeLimitInterceptor(
  call: grpc.ServerUnaryCall<any, any> | grpc.ServerWritableStream<any, any>,
  callback: grpc.sendUnaryData<any> | undefined,
  next: () => void
): void {
  const method = call.getPath();
  const maxSize = methodSizeLimits[method] || MAX_REQUEST_SIZE;

  // Get request size (approximate)
  const requestSize = JSON.stringify(call.request).length;

  if (requestSize > maxSize) {
    logger.warn('Request size limit exceeded', {
      method,
      requestSize,
      maxSize,
    });

    const error = new GrpcError(
      grpc.status.INVALID_ARGUMENT,
      `Request size (${requestSize} bytes) exceeds maximum (${maxSize} bytes)`,
      { requestSize, maxSize }
    );

    if (callback) {
      return callback(error.toServiceError());
    } else {
      (call as grpc.ServerWritableStream<any, any>).destroy(
        error.toServiceError()
      );
      return;
    }
  }

  next();
}

5. Correlation ID Interceptor

// src/grpc/interceptors/correlation.interceptor.ts
import * as grpc from '@grpc/grpc-js';
import { v4 as uuidv4 } from 'uuid';

declare module '@grpc/grpc-js' {
  interface ServerUnaryCall<RequestType, ResponseType> {
    correlationId?: string;
  }
  interface ServerWritableStream<RequestType, ResponseType> {
    correlationId?: string;
  }
}

export function correlationInterceptor(
  call: grpc.ServerUnaryCall<any, any> | grpc.ServerWritableStream<any, any>,
  callback: grpc.sendUnaryData<any> | undefined,
  next: () => void
): void {
  // Extract or generate correlation ID
  const metadata = call.metadata;
  let correlationId = metadata.get('x-correlation-id')[0]?.toString();

  if (!correlationId) {
    correlationId = uuidv4();
  }

  // Attach to call
  call.correlationId = correlationId;

  // Add to response metadata
  if (callback) {
    const originalCallback = callback;

    callback = (error, response) => {
      const responseMetadata = new grpc.Metadata();
      responseMetadata.add('x-correlation-id', correlationId!);
      
      call.sendMetadata(responseMetadata);
      
      originalCallback(error, response);
    };
  }

  next();
}

Retry Logic (Client-Side)

// src/client/retry-client.ts
import * as grpc from '@grpc/grpc-js';
import { logger } from '@config/logger';

interface RetryConfig {
  maxRetries: number;
  initialBackoffMs: number;
  maxBackoffMs: number;
  retryableStatuses: grpc.status[];
}

const defaultRetryConfig: RetryConfig = {
  maxRetries: 3,
  initialBackoffMs: 100,
  maxBackoffMs: 5000,
  retryableStatuses: [
    grpc.status.UNAVAILABLE,
    grpc.status.DEADLINE_EXCEEDED,
    grpc.status.RESOURCE_EXHAUSTED,
  ],
};

export class RetryableGrpcClient {
  constructor(private config: RetryConfig = defaultRetryConfig) {}

  async callWithRetry<TRequest, TResponse>(
    fn: (request: TRequest) => Promise<TResponse>,
    request: TRequest
  ): Promise<TResponse> {
    let lastError: any;
    let backoffMs = this.config.initialBackoffMs;

    for (let attempt = 0; attempt <= this.config.maxRetries; attempt++) {
      try {
        return await fn(request);
      } catch (error: any) {
        lastError = error;

        // Check if error is retryable
        if (!this.config.retryableStatuses.includes(error.code)) {
          throw error;
        }

        // Don't retry on last attempt
        if (attempt === this.config.maxRetries) {
          break;
        }

        logger.warn('gRPC call failed, retrying', {
          attempt: attempt + 1,
          maxRetries: this.config.maxRetries,
          backoffMs,
          errorCode: error.code,
        });

        // Wait before retry with exponential backoff
        await this.sleep(backoffMs);
        
        // Increase backoff for next retry
        backoffMs = Math.min(backoffMs * 2, this.config.maxBackoffMs);
      }
    }

    logger.error('gRPC call failed after all retries', {
      maxRetries: this.config.maxRetries,
      lastError,
    });

    throw lastError;
  }

  private sleep(ms: number): Promise<void> {
    return new Promise(resolve => setTimeout(resolve, ms));
  }
}

// Usage example
const retryClient = new RetryableGrpcClient();

const order = await retryClient.callWithRetry(
  (request) => orderClient.getOrder(request),
  { id: 'ord_123' }
);

Circuit Breaker Pattern

// src/client/circuit-breaker.ts
import { logger } from '@config/logger';

enum CircuitState {
  CLOSED = 'CLOSED', // Normal operation
  OPEN = 'OPEN', // Failures exceeded, rejecting requests
  HALF_OPEN = 'HALF_OPEN', // Testing if service recovered
}

interface CircuitBreakerConfig {
  failureThreshold: number; // Failures before opening circuit
  successThreshold: number; // Successes to close circuit from half-open
  timeout: number; // Time to wait before trying half-open (ms)
}

export class CircuitBreaker {
  private state: CircuitState = CircuitState.CLOSED;
  private failureCount: number = 0;
  private successCount: number = 0;
  private nextAttempt: number = Date.now();

  constructor(
    private serviceName: string,
    private config: CircuitBreakerConfig = {
      failureThreshold: 5,
      successThreshold: 2,
      timeout: 60000, // 1 minute
    }
  ) {}

  async execute<T>(fn: () => Promise<T>): Promise<T> {
    if (this.state === CircuitState.OPEN) {
      if (Date.now() < this.nextAttempt) {
        throw new Error(
          `Circuit breaker is OPEN for ${this.serviceName}`
        );
      }

      // Try half-open
      this.state = CircuitState.HALF_OPEN;
      logger.info('Circuit breaker transitioning to HALF_OPEN', {
        service: this.serviceName,
      });
    }

    try {
      const result = await fn();
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      throw error;
    }
  }

  private onSuccess(): void {
    this.failureCount = 0;

    if (this.state === CircuitState.HALF_OPEN) {
      this.successCount++;

      if (this.successCount >= this.config.successThreshold) {
        this.state = CircuitState.CLOSED;
        this.successCount = 0;
        
        logger.info('Circuit breaker CLOSED', {
          service: this.serviceName,
        });
      }
    }
  }

  private onFailure(): void {
    this.failureCount++;
    this.successCount = 0;

    if (this.failureCount >= this.config.failureThreshold) {
      this.state = CircuitState.OPEN;
      this.nextAttempt = Date.now() + this.config.timeout;

      logger.error('Circuit breaker OPENED', {
        service: this.serviceName,
        failureCount: this.failureCount,
      });
    }
  }

  getState(): CircuitState {
    return this.state;
  }
}

// Usage
const orderServiceBreaker = new CircuitBreaker('OrderService');

const order = await orderServiceBreaker.execute(() =>
  orderClient.getOrder({ id: 'ord_123' })
);

Complete Interceptor Chain

// src/grpc/server.ts
import * as grpc from '@grpc/grpc-js';
import { correlationInterceptor } from './interceptors/correlation.interceptor';
import { loggingInterceptor } from './interceptors/logging.interceptor';
import { metricsInterceptor } from './interceptors/metrics.interceptor';
import { sizeLimitInterceptor } from './interceptors/size-limit.interceptor';
import { timeoutInterceptor } from './interceptors/timeout.interceptor';
import { apiKeyInterceptor } from './interceptors/api-key.interceptor';
import { authInterceptor } from './interceptors/auth.interceptor';
import { authorizationInterceptor } from './interceptors/authorization.interceptor';
import { rateLimitInterceptor } from './interceptors/rate-limit.interceptor';
import { errorInterceptor } from './interceptors/error.interceptor';

export class GrpcServer {
  private server: grpc.Server;

  constructor() {
    this.server = new grpc.Server({
      // Interceptors execute in this order
      interceptors: [
        correlationInterceptor, // 1. Generate correlation ID
        loggingInterceptor, // 2. Log request
        metricsInterceptor, // 3. Collect metrics
        sizeLimitInterceptor, // 4. Check request size
        timeoutInterceptor, // 5. Set timeout
        apiKeyInterceptor, // 6. API key auth
        authInterceptor, // 7. JWT auth
        authorizationInterceptor, // 8. Check permissions
        rateLimitInterceptor, // 9. Rate limiting
        errorInterceptor, // 10. Handle errors
      ],
      'grpc.max_receive_message_length': 10 * 1024 * 1024,
      'grpc.max_send_message_length': 10 * 1024 * 1024,
    });

    this.registerServices();
  }

  // ... rest of implementation
}

Key Takeaways

1. Proper Error Codes: Use appropriate gRPC status codes, not just INTERNAL

2. Rich Error Details: Include request IDs and context for debugging

3. Validation First: Validate all inputs before processing

4. Interceptor Chain: Use interceptors for cross-cutting concerns

5. Never Expose Internals: Generic errors to clients, detailed logs on server

6. Retry + Circuit Breaker: Client-side resilience patterns

7. Observability: Log correlation IDs, metrics, and traces

My 3 AM Lesson: Good error handling is invisible when things work, but saves your company when things break.

---

Next: Part 6: API Documentation and Versioning in gRPC

Series Navigation: gRPC 101 Series