Complete Pipeline

Copy pipeline YAML

Review it for your environment before you deploy.

This pipeline combines all encryption patterns from this tutorial:

• Payment encryption - Card numbers and CVV with environment-provided keys
• PII encryption - Emails, phones, and SSNs
• Address encryption - Streets and detailed ZIPs
• Temporal encryption - Birth dates with age-range preservation
• Multi-key architecture - Risk-based key tiers for breach containment

Payment, customer, address, and birth-date fields use separate environment-provided keys. Each encrypted value carries its own base64-encoded 12-byte nonce, while the mappings retain limited analytics fields such as card last four, email domain, ZIP prefix, birth year, and age range. The integrating system owns key rotation and authorization.

Full Configuration

Complete pipeline

encryption-patterns.yaml

name: encryption-patterns-complete
type: pipeline
description: Field-level encryption example with configurable patterns relevant to security and privacy programs; applicability and compliance require independent review.
namespace: production
labels:
  category: data-security
  pattern: encryption
  compliance: pci-gdpr-hipaa

config:
  input:
    http_server:
      address: '0.0.0.0:8080'
      path: '/encrypt'
      allowed_verbs: ['POST']
      timeout: '30s'

  pipeline:
    processors:
      # Input validation
      - mapping: |
          let required_fields = ["payment.card_number", "customer.email"]
          let missing_fields = $required_fields.filter(field -> this.get(field) == null)

          root = if $missing_fields.length() > 0 {
            throw("Missing required fields: " + $missing_fields.join(", "))
          } else {
            this
          }

          root.processing_metadata = {
            "pipeline_version": env("PIPELINE_VERSION").or("2.1.0"),
            "node_id": env("NODE_ID").or("unknown"),
            "processing_start": now()
          }

      # PATTERN 1: Payment Card Encryption (PCI-DSS)
      - mapping: |
          root = this

          let key = env("PAYMENT_ENCRYPTION_KEY").or(throw("PAYMENT_ENCRYPTION_KEY is not set"))
          let card_number_iv = uuid_v4().hash("sha256").slice(0, 12)
          let cvv_iv = uuid_v4().hash("sha256").slice(0, 12)
          let card_num = this.payment.card_number.or("").string().re_replace_all("[^0-9]", "")

          root.payment.card_brand = if this.exists("payment.card_number") {
            match {
              $card_num.re_match("^4[0-9]{12,18}$") => "visa"
              $card_num.re_match("^5[1-5][0-9]{14}$") => "mastercard"
              $card_num.re_match("^3[47][0-9]{13}$") => "american_express"
              $card_num.re_match("^6011[0-9]{12}$") => "discover"
              _ => "unknown"
            }
          }

          root.payment.card_last_four = if $card_num.length() >= 8 { $card_num.slice(-4) }
          root.payment.card_bin = if $card_num.length() >= 8 { $card_num.slice(0, 6) }

          root.payment.card_number_nonce = if this.exists("payment.card_number") { $card_number_iv.encode("base64") }

          root.payment.card_number_encrypted = if this.exists("payment.card_number") {
            this.payment.card_number.string().encrypt_aes("gcm", $key, $card_number_iv).encode("base64")
          }

          root.payment.cvv_nonce = if this.exists("payment.cvv") { $cvv_iv.encode("base64") }

          root.payment.cvv_encrypted = if this.exists("payment.cvv") {
            this.payment.cvv.string().encrypt_aes("gcm", $key, $cvv_iv).encode("base64")
          }

          root.payment = root.payment.without("card_number", "cvv", "cardholder_name")

      # PATTERN 2: Personal Data Encryption (GDPR/CCPA)
      - mapping: |
          root = this

          let key = env("PII_ENCRYPTION_KEY").or(throw("PII_ENCRYPTION_KEY is not set"))
          let email_iv = uuid_v4().hash("sha256").slice(0, 12)
          let phone_iv = uuid_v4().hash("sha256").slice(0, 12)
          let ssn_iv = uuid_v4().hash("sha256").slice(0, 12)
          let email = this.customer.email.or("").string().lowercase()
          let ssn_normalized = this.customer.ssn.or("").string().re_replace_all("[^0-9]", "")

          root.customer.email_domain = if $email.re_match("^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$") {
            $email.split("@").index(1)
          }

          root.customer.email_nonce = if this.exists("customer.email") { $email_iv.encode("base64") }

          root.customer.email_encrypted = if this.exists("customer.email") {
            this.customer.email.string().encrypt_aes("gcm", $key, $email_iv).encode("base64")
          }

          root.customer.phone_nonce = if this.exists("customer.phone") { $phone_iv.encode("base64") }

          root.customer.phone_encrypted = if this.exists("customer.phone") {
            this.customer.phone.string().encrypt_aes("gcm", $key, $phone_iv).encode("base64")
          }

          root.customer.ssn_last_four = if $ssn_normalized.length() == 9 {
            $ssn_normalized.slice(-4)
          }

          root.customer.ssn_nonce = if this.exists("customer.ssn") { $ssn_iv.encode("base64") }

          root.customer.ssn_encrypted = if this.exists("customer.ssn") {
            this.customer.ssn.string().encrypt_aes("gcm", $key, $ssn_iv).encode("base64")
          }

          root.customer = root.customer.without("email", "phone", "ssn", "first_name", "last_name")

      # PATTERN 3: Address Encryption
      - mapping: |
          root = this

          let key = env("ADDRESS_ENCRYPTION_KEY").or(throw("ADDRESS_ENCRYPTION_KEY is not set"))
          let street_iv = uuid_v4().hash("sha256").slice(0, 12)
          let zip_iv = uuid_v4().hash("sha256").slice(0, 12)
          let zip_clean = this.billing_address.zip.or("").string().re_replace_all("[^0-9]", "")

          root.billing_address.zip_prefix = if $zip_clean.length() >= 3 { $zip_clean.slice(0, 3) }

          root.billing_address.street_nonce = if this.exists("billing_address.street") { $street_iv.encode("base64") }

          root.billing_address.street_encrypted = if this.exists("billing_address.street") {
            this.billing_address.street.string().encrypt_aes("gcm", $key, $street_iv).encode("base64")
          }

          root.billing_address.zip_nonce = if this.exists("billing_address.zip") { $zip_iv.encode("base64") }

          root.billing_address.zip_encrypted = if this.exists("billing_address.zip") {
            this.billing_address.zip.string().encrypt_aes("gcm", $key, $zip_iv).encode("base64")
          }

          root.billing_address = root.billing_address.without("street", "zip")

      # PATTERN 4: Temporal Data Encryption (HIPAA)
      - mapping: |
          root = this

          let key = env("TEMPORAL_ENCRYPTION_KEY").or(throw("TEMPORAL_ENCRYPTION_KEY is not set"))
          let date_of_birth_iv = uuid_v4().hash("sha256").slice(0, 12)
          let has_dob = this.exists("customer.date_of_birth")
          let birth_year = if $has_dob {
            this.customer.date_of_birth.ts_parse("2006-01-02").ts_format("2006").number()
          }
          let current_year = now().ts_format("2006").number()
          let age = if $has_dob { $current_year - $birth_year }

          root.customer.birth_year = $birth_year

          root.customer.age_range = if root.customer.birth_year != null {
            match {
              $age < 18 => "under_18"
              $age < 35 => "18_to_34"
              $age < 55 => "35_to_54"
              $age < 65 => "55_to_64"
              _ => "65_plus"
            }
          }

          root.customer.date_of_birth_nonce = if $has_dob { $date_of_birth_iv.encode("base64") }

          root.customer.date_of_birth_encrypted = if $has_dob {
            this.customer.date_of_birth.string().encrypt_aes("gcm", $key, $date_of_birth_iv).encode("base64")
          }
          root.customer.date_of_birth_nonce = if $has_dob { $date_of_birth_iv.encode("base64") }

          root.customer.date_of_birth = deleted()

      # Final validation
      - mapping: |
          root = this

          let validation_errors = [
            if this.exists("payment.card_number") { "payment.card_number still in plaintext" },
            if this.exists("customer.email") { "customer.email still in plaintext" },
            if this.exists("customer.ssn") { "customer.ssn still in plaintext" }
          ].filter(v -> v != null)

          root = if $validation_errors.length() > 0 {
            throw("Encryption validation failed: " + $validation_errors.join(", "))
          } else {
            this
          }

          root.processing_metadata.processing_complete = now()
          root.processing_metadata.encryption_status = "success"
      - catch:
          - mapping: root = deleted()

  output:
    broker:
      pattern: fan_out
      outputs:
        # Primary encrypted data
        - file:
            path: './encrypted-data-${!timestamp_unix()}.jsonl'
            codec: 'lines'

        # Audit trail
        - file:
            path: './audit-trail-${!timestamp_unix()}.jsonl'
            codec: 'lines'
          processors:
            - mapping: |
                root = {
                  "audit_id": "enc_" + uuid_v4(),
                  "timestamp": now(),
                  "event_type": "field_level_encryption_complete",
                  "node_id": env("NODE_ID").or("unknown"),
                  "encryption_summary": {
                    "fields_encrypted": 8,
                    "compliance_standards_met": ["PCI-DSS", "GDPR", "CCPA", "HIPAA"]
                  }
                }

logger:
  level: 'INFO'
  format: json

metrics:
  prometheus:
    path: '/metrics'

Quick Test

# Send transaction with multiple sensitive field types
curl -X POST http://localhost:8080/transactions \
  -H "Content-Type: application/json" \
  -d '{
    "transaction_id": "txn_001",
    "card_number": "4532123456789010",
    "cvv": "123",
    "email": "[email protected]",
    "phone": "415-555-1234",
    "ssn": "123-45-6789",
    "birth_date": "1985-03-15",
    "address": {
      "street": "123 Main St",
      "city": "San Francisco",
      "state": "CA",
      "zip": "94102"
    }
  }'

# Each field type encrypted with its appropriate key tier
# Analytics-safe summaries preserved (last_four, domain, age_range, etc.)

Deploy

# Deploy to Expanso orchestrator
expanso-cli job deploy encryption-patterns.yaml

# Or run locally with expanso-edge
expanso-edge run --config encryption-patterns.yaml

Download

Download encryption-patterns.yaml

What's Next?

• Troubleshooting - Common issues and solutions
• Enforce Schema - Validate data structure