Node.js / TypeScript Data Provider

This guide shows how to build a GW data provider using Express and TypeScript. Two patterns are demonstrated:

• Middleware pattern -- a reusable wrapper handles envelope parsing, delivery, and callbacks automatically
• Manual pattern -- you control every protocol step for maximum flexibility

Project Setup

npm init -y
npm install express @aws-sdk/client-s3
npm install -D typescript @types/express @types/node

tsconfig.json

{
  "compilerOptions": {
    "target": "ES2022",
    "module": "NodeNext",
    "moduleResolution": "NodeNext",
    "outDir": "dist",
    "strict": true
  }
}

Server Entry Point

src/server.ts

import express, { Request, Response, NextFunction } from 'express';

const PORT = parseInt(process.env.PORT ?? '3002', 10);
const GW_API_KEY = process.env.GW_API_KEY ?? 'demo-api-key';

const app = express();
app.use(express.json());

// API-key validation middleware
function validateApiKey(req: Request, res: Response, next: NextFunction): void {
  if (req.path === '/health') {
    next();
    return;
  }

  const key = req.headers['x-gw-api-key'];
  if (key !== GW_API_KEY) {
    res.status(401).json({ error: 'Invalid or missing X-GW-Api-Key header' });
    return;
  }
  next();
}

app.use(validateApiKey);

// Health check
app.get('/health', (_req: Request, res: Response) => {
  res.json({ status: 'ok', service: 'data-provider' });
});

app.listen(PORT, () => {
  console.log(`[data-provider] listening on :${PORT}`);
});

Query Envelope

Parse and validate the incoming query envelope from the platform:

src/envelope.ts

export interface DeliverySpec {
  mechanism: 'gw_s3_sync' | 'webhook' | 'inline';
  url?: string;
  s3Bucket?: string;
  s3KeyPrefix?: string;
}

export interface QueryEnvelope {
  queryId: string;
  datasetId: string;
  endpoint: string;
  parameters: Record<string, unknown>;
  delivery: DeliverySpec;
  callbackUrl: string;
  callbackToken: string;
}

export class EnvelopeError extends Error {
  constructor(message: string) {
    super(message);
    this.name = 'EnvelopeError';
  }
}

const REQUIRED_FIELDS: (keyof QueryEnvelope)[] = [
  'queryId', 'datasetId', 'endpoint', 'parameters',
  'delivery', 'callbackUrl', 'callbackToken',
];

export function parseEnvelope(body: unknown): QueryEnvelope {
  if (!body || typeof body !== 'object') {
    throw new EnvelopeError('Request body must be a JSON object');
  }

  const obj = body as Record<string, unknown>;

  for (const field of REQUIRED_FIELDS) {
    if (obj[field] === undefined || obj[field] === null) {
      throw new EnvelopeError(`Missing required envelope field: ${field}`);
    }
  }

  const delivery = obj.delivery as Record<string, unknown>;
  if (!delivery?.mechanism || typeof delivery.mechanism !== 'string') {
    throw new EnvelopeError('delivery.mechanism is required');
  }

  return {
    queryId: String(obj.queryId),
    datasetId: String(obj.datasetId),
    endpoint: String(obj.endpoint),
    parameters: obj.parameters as Record<string, unknown>,
    delivery: {
      mechanism: delivery.mechanism as DeliverySpec['mechanism'],
      url: delivery.url ? String(delivery.url) : undefined,
      s3Bucket: delivery.s3Bucket ? String(delivery.s3Bucket) : undefined,
      s3KeyPrefix: delivery.s3KeyPrefix ? String(delivery.s3KeyPrefix) : undefined,
    },
    callbackUrl: String(obj.callbackUrl),
    callbackToken: String(obj.callbackToken),
  };
}

Result Delivery

Route results to S3, a webhook, or return them inline:

src/delivery.ts

import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
import type { QueryEnvelope } from './envelope.js';

export interface DeliveryResult {
  mechanism: string;
  reference?: string;
  statusCode?: number;
}

export interface ProviderConfig {
  s3Endpoint: string;
  s3Bucket: string;
  gwApiKey: string;
  gwCallbackSecret: string;
}

export async function deliverResults(
  envelope: QueryEnvelope,
  results: unknown,
  config: ProviderConfig,
): Promise<DeliveryResult> {
  switch (envelope.delivery.mechanism) {
    case 'gw_s3_sync':
      return deliverToS3(envelope, results, config);
    case 'webhook':
      return deliverToWebhook(envelope, results);
    case 'inline':
    default:
      return { mechanism: 'inline' };
  }
}

async function deliverToS3(
  envelope: QueryEnvelope,
  results: unknown,
  config: ProviderConfig,
): Promise<DeliveryResult> {
  const bucket = envelope.delivery.s3Bucket ?? config.s3Bucket;
  const prefix = envelope.delivery.s3KeyPrefix ?? '';
  const key = `${prefix}${envelope.queryId}.json`;

  const s3 = new S3Client({
    endpoint: config.s3Endpoint,
    region: 'us-east-1',
    forcePathStyle: true,
  });

  await s3.send(new PutObjectCommand({
    Bucket: bucket,
    Key: key,
    Body: JSON.stringify(results),
    ContentType: 'application/json',
  }));

  return { mechanism: 'gw_s3_sync', reference: `s3://${bucket}/${key}` };
}

async function deliverToWebhook(
  envelope: QueryEnvelope,
  results: unknown,
): Promise<DeliveryResult> {
  const url = envelope.delivery.url;
  if (!url) throw new Error('Webhook delivery requires delivery.url');

  const res = await fetch(url, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ queryId: envelope.queryId, results }),
  });

  return { mechanism: 'webhook', statusCode: res.status };
}

HMAC-Signed Callback

Post the query status back to the platform with an HMAC-SHA256 signature:

src/callback.ts

import { createHmac } from 'crypto';
import type { QueryEnvelope } from './envelope.js';
import type { DeliveryResult } from './delivery.js';

export interface CallbackPayload {
  queryId: string;
  status: 'delivered' | 'failed' | 'partial';
  recordCount: number;
  executionTimeMs: number;
  delivery?: DeliveryResult;
  error?: string;
}

export async function postCallback(
  envelope: QueryEnvelope,
  info: {
    recordCount: number;
    status: 'delivered' | 'failed' | 'partial';
    executionTimeMs: number;
    delivery?: DeliveryResult;
    error?: string;
  },
): Promise<void> {
  const body: CallbackPayload = {
    queryId: envelope.queryId,
    status: info.status,
    recordCount: info.recordCount,
    executionTimeMs: info.executionTimeMs,
    delivery: info.delivery,
    error: info.error,
  };

  const payload = JSON.stringify(body);

  const signature = createHmac('sha256', envelope.callbackToken)
    .update(payload)
    .digest('hex');

  try {
    await fetch(envelope.callbackUrl, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'X-GW-Signature': `sha256=${signature}`,
      },
      body: payload,
    });
  } catch (err) {
    // Log but don't throw -- the query succeeded even if the callback fails.
    // The platform will retry or poll for status.
    console.error(
      `[callback] Failed to POST callback for query ${envelope.queryId}:`,
      err instanceof Error ? err.message : err,
    );
  }
}

Middleware Pattern

The middleware pattern wraps the protocol plumbing so your handler only contains business logic:

src/routes/companies.ts

import { Router, Request, Response } from 'express';
import { parseEnvelope, EnvelopeError, type QueryEnvelope } from '../envelope.js';
import { deliverResults, type ProviderConfig } from '../delivery.js';
import { postCallback } from '../callback.js';

type BusinessLogicHandler = (
  params: Record<string, unknown>,
  envelope: QueryEnvelope,
) => unknown[] | Promise<unknown[]>;

function gwMiddleware(config: ProviderConfig, handler: BusinessLogicHandler) {
  return async (req: Request, res: Response): Promise<void> => {
    const startTime = Date.now();
    let envelope: QueryEnvelope;

    // 1. Parse envelope
    try {
      envelope = parseEnvelope(req.body);
    } catch (err) {
      const message = err instanceof EnvelopeError ? err.message : 'Invalid envelope';
      res.status(400).json({ error: message });
      return;
    }

    try {
      // 2. Run business logic
      const results = await handler(envelope.parameters, envelope);

      // 3. Deliver results
      const delivery = await deliverResults(envelope, results, config);
      const executionTimeMs = Date.now() - startTime;

      // 4. POST callback to GW
      await postCallback(envelope, {
        recordCount: results.length,
        status: 'delivered',
        executionTimeMs,
        delivery,
      });

      // Respond to the original request
      if (delivery.mechanism === 'inline') {
        res.json({ queryId: envelope.queryId, recordCount: results.length, executionTimeMs, results });
      } else {
        res.json({ queryId: envelope.queryId, recordCount: results.length, executionTimeMs, delivery });
      }
    } catch (err) {
      const executionTimeMs = Date.now() - startTime;
      const message = err instanceof Error ? err.message : 'Internal error';

      await postCallback(envelope, {
        recordCount: 0,
        status: 'failed',
        executionTimeMs,
        error: message,
      });

      res.status(500).json({ error: message, queryId: envelope.queryId });
    }
  };
}

export function createCompaniesRouter(config: ProviderConfig): Router {
  const router = Router();

  // Your handler only contains business logic
  router.post('/', gwMiddleware(config, (params) => {
    return searchCompanies({
      name: params.name as string | undefined,
      country: params.country as string | undefined,
    });
  }));

  return router;
}

Mount the router in your server:

app.use('/api/search/companies', createCompaniesRouter(config));

Manual Pattern

For endpoints that need explicit control over each step:

src/routes/sanctions.ts

import { Router, Request, Response } from 'express';
import { parseEnvelope, EnvelopeError } from '../envelope.js';
import { deliverResults, type ProviderConfig } from '../delivery.js';
import { postCallback } from '../callback.js';

export function createSanctionsRouter(config: ProviderConfig): Router {
  const router = Router();

  router.post('/', async (req: Request, res: Response): Promise<void> => {
    const startTime = Date.now();

    // Step 1: Manually parse the envelope
    let envelope;
    try {
      envelope = parseEnvelope(req.body);
    } catch (err) {
      const message = err instanceof EnvelopeError ? err.message : 'Invalid envelope';
      res.status(400).json({ error: message });
      return;
    }

    // Step 2: Run business logic (with custom validation)
    const name = envelope.parameters.name as string | undefined;
    if (!name) {
      res.status(400).json({
        error: 'parameters.name is required for sanctions checks',
        queryId: envelope.queryId,
      });
      return;
    }

    const results = checkSanctions({ name });
    const executionTimeMs = Date.now() - startTime;

    // Step 3: Explicitly deliver results
    try {
      const delivery = await deliverResults(envelope, results, config);

      // Step 4: Explicitly post callback
      await postCallback(envelope, {
        recordCount: results.length,
        status: 'delivered',
        executionTimeMs,
        delivery,
      });

      if (delivery.mechanism === 'inline') {
        res.json({ queryId: envelope.queryId, recordCount: results.length, executionTimeMs, results });
      } else {
        res.json({ queryId: envelope.queryId, recordCount: results.length, executionTimeMs, delivery });
      }
    } catch (err) {
      const message = err instanceof Error ? err.message : 'Delivery failed';

      await postCallback(envelope, {
        recordCount: 0,
        status: 'failed',
        executionTimeMs: Date.now() - startTime,
        error: message,
      });

      res.status(500).json({ error: message, queryId: envelope.queryId });
    }
  });

  return router;
}

When to Use Each Pattern

Pattern	Use When
Middleware	Standard query-response endpoints; you just need to run a search and return results
Manual	Custom parameter validation, conditional delivery, streaming results, partial responses, or complex error recovery

Testing Locally

Use the Docker Compose setup from the Delivery Targets guide to run LocalStack and mock webhook receivers, then call your provider:

curl -X POST http://localhost:3002/api/search/companies \
  -H "Content-Type: application/json" \
  -H "X-GW-Api-Key: demo-api-key" \
  -d '{
    "queryId": "test-001",
    "datasetId": "ds-companies",
    "endpoint": "/api/search/companies",
    "parameters": { "name": "Acme" },
    "delivery": { "mechanism": "inline" },
    "callbackUrl": "http://localhost:8083/webhook",
    "callbackToken": "test-secret"
  }'