Skip to content

Complete Recipe Examples

This document provides comprehensive examples of complete YAML recipe configurations, demonstrating how to combine multiple metric types and leverage fan-out expansion effectively.

Basic Multi-Metric Recipe

datasets:
  validation_set:
    type: csv
    source: data/validation.csv
  loan_portfolio:
    type: csv
    source: data/loans.csv
  customer_data:
    type: csv
    source: data/customers.csv
  test_set:
    type: csv
    source: data/test_data.csv
  prediction_data:
    type: csv
    source: data/predictions.csv
collections:
  model_performance:
    metrics:
    - name:
      - model_accuracy
      prob_def: predicted_default
      default: actual_default
      metric_type: default_accuracy
      data_format: record
    dataset: validation_set
  loan_statistics:
    metrics:
    - name:
      - avg_loan_amount
      variable: loan_amount
      metric_type: mean
      data_format: record
    dataset: loan_portfolio
  credit_summary:
    metrics:
    - name:
      - median_credit_score
      variable: credit_score
      metric_type: median
      data_format: record
    dataset: customer_data
  discrimination_test:
    metrics:
    - name:
      - model_auc
      prob_def: default_probability
      default: is_default
      metric_type: auc
      data_format: record
    dataset: test_set
  gini_coefficient:
    metrics:
    - name:
      - model_gini
      prob_def: default_probability
      default: is_default
      metric_type: gini
      data_format: record
    dataset: test_set
  prediction_accuracy:
    metrics:
    - name:
      - ead_mape
      observed: actual_ead
      predicted: predicted_ead
      metric_type: mape
      data_format: record
    dataset: prediction_data

Advanced Fan-out Examples

Multi-Segment Performance Analysis

datasets:
  validation_data:
    type: csv
    source: data/model_validation.csv
  exposure_data:
    type: csv
    source: data/exposures.csv
collections:
  multi_model_accuracy:
    metrics:
    - name:
      - model_v1_accuracy
      - model_v2_accuracy
      - model_v3_accuracy
      segment:
      - - prime
      - - near_prime
      - - subprime
      prob_def: predicted_default
      default: actual_default
      metric_type: default_accuracy
      data_format: record
    dataset: validation_data
  ead_confidence_analysis:
    metrics:
    - name:
      - ead_corporate
      - ead_retail
      - ead_sme
      segment:
      - - corporate
      - - retail
      - - sme
      predicted_ead: predicted_ead
      actual_ead: actual_ead
      default: default_flag
      metric_type: ead_accuracy
      data_format: record
    dataset: exposure_data
  prediction_error_analysis:
    metrics:
    - name:
      - mape_corporate
      - mape_retail
      - mape_sme
      segment:
      - - corporate
      - - retail
      - - sme
      observed: actual_ead
      predicted: predicted_ead
      metric_type: mape
      data_format: record
    dataset: exposure_data

Comprehensive Model Validation

datasets:
  performance_data:
    type: csv
    source: data/monthly_quarterly_performance.csv
  calibration_data:
    type: csv
    source: data/calibration_sample.csv
  current_month:
    type: csv
    source: data/current_scores.csv
  baseline_month:
    type: csv
    source: data/baseline_scores.csv
collections:
  discrimination_metrics:
    metrics:
    - name:
      - high_risk_auc
      - medium_risk_auc
      - low_risk_auc
      prob_def: default_probability
      default: default_flag
      segment:
      - - high_risk
      - - medium_risk
      - - low_risk
      metric_type: auc
      data_format: record
    dataset: performance_data
  gini_coefficients:
    metrics:
    - name:
      - high_risk_gini
      - medium_risk_gini
      - low_risk_gini
      prob_def: default_probability
      default: default_flag
      segment:
      - - high_risk
      - - medium_risk
      - - low_risk
      metric_type: gini
      data_format: record
    dataset: performance_data
  calibration_testing:
    metrics:
    - name:
      - calibration_test
      prob_def: default_probability
      default: default_outcome
      bands: 10
      metric_type: hosmer_lemeshow
      data_format: record
    dataset: calibration_data
  distribution_stability:
    metrics:
    - name:
      - score_stability_new
      - score_stability_existing
      variable: credit_score
      segment:
      - - new_customers
      - - existing_customers
      metric_type: jeffreys_test
      data_format: record
    dataset: current_month

Time Series Monitoring

datasets:
  monthly_data:
    type: csv
    source: data/monthly_performance.csv
  monthly_scores:
    type: csv
    source: data/monthly_risk_scores.csv
  baseline_scores:
    type: csv
    source: data/baseline_risk_scores.csv
  portfolio_data:
    type: csv
    source: data/portfolio_exposures.csv
collections:
  monthly_accuracy:
    metrics:
    - name:
      - jan_accuracy
      - feb_accuracy
      - mar_accuracy
      - apr_accuracy
      segment:
      - - jan
      - - feb
      - - mar
      - - apr
      prob_def: model_prediction
      default: observed_default
      metric_type: default_accuracy
      data_format: record
    dataset: monthly_data
  score_stability:
    metrics:
    - name:
      - jan_stability
      - feb_stability
      - mar_stability
      segment:
      - - jan
      - - feb
      - - mar
      variable: risk_score
      metric_type: jeffreys_test
      data_format: record
    dataset: monthly_scores
  exposure_summaries:
    metrics:
    - name:
      - jan_avg_exposure
      - feb_avg_exposure
      - mar_avg_exposure
      - apr_avg_exposure
      segment:
      - - jan
      - - feb
      - - mar
      - - apr
      variable: exposure_amount
      metric_type: mean
      data_format: record
    dataset: portfolio_data
  exposure_medians:
    metrics:
    - name:
      - jan_med_exposure
      - feb_med_exposure
      - mar_med_exposure
      - apr_med_exposure
      segment:
      - - jan
      - - feb
      - - mar
      - - apr
      variable: exposure_amount
      metric_type: median
      data_format: record
    dataset: portfolio_data

A/B Testing Configuration

datasets:
  ab_test_results:
    type: csv
    source: data/ab_test_complete.csv
  treatment_data:
    type: csv
    source: data/treatment_scores.csv
  control_data:
    type: csv
    source: data/control_scores.csv
collections:
  ab_accuracy_comparison:
    metrics:
    - name:
      - control_accuracy
      - treatment_accuracy
      segment:
      - - control_group
      - - treatment_group
      prob_def: model_prediction
      default: actual_outcome
      metric_type: default_accuracy
      data_format: record
    dataset: ab_test_results
  ab_auc_comparison:
    metrics:
    - name:
      - control_auc
      - treatment_auc
      segment:
      - - control_group
      - - treatment_group
      prob_def: risk_probability
      default: default_flag
      metric_type: auc
      data_format: record
    dataset: ab_test_results
  distribution_analysis:
    metrics:
    - name:
      - treatment_vs_control
      variable: risk_score
      metric_type: jeffreys_test
      data_format: record
    dataset: treatment_data

Comprehensive Discrimination Analysis

datasets:
  validation_set:
    type: csv
    source: data/model_validation.csv
  grade_summary:
    type: csv
    source: data/risk_grade_performance.csv
collections:
  overall_discrimination:
    metrics:
    - name:
      - overall_model_auc
      data_format: record
      prob_def: model_probability
      default: default_indicator
      metric_type: auc
    dataset: validation_set
  overall_gini:
    metrics:
    - name:
      - overall_model_gini
      data_format: record
      prob_def: model_probability
      default: default_indicator
      metric_type: gini
    dataset: validation_set
  segmented_auc:
    metrics:
    - name:
      - product_auc
      - region_auc
      - vintage_auc
      data_format: record
      prob_def: model_probability
      default: default_indicator
      segment:
      - - product_type
      - - region
      - - origination_year
      metric_type: auc
    dataset: validation_set
  segmented_gini:
    metrics:
    - name:
      - product_gini
      - region_gini
      - vintage_gini
      data_format: record
      prob_def: model_probability
      default: default_indicator
      segment:
      - - product_type
      - - region
      - - origination_year
      metric_type: gini
    dataset: validation_set
  ks_statistics:
    metrics:
    - name:
      - overall_ks
      - product_ks
      data_format: record
      prob_def: model_probability
      default: default_indicator
      segment:
      - null
      - - product_type
      metric_type: kolmogorov_smirnov
    dataset: validation_set
  grade_level_auc:
    metrics:
    - name:
      - risk_grade_auc
      data_format: summary
      mean_pd: avg_probability
      defaults: default_count
      volume: total_count
      metric_type: auc
    dataset: grade_summary
  grade_level_gini:
    metrics:
    - name:
      - risk_grade_gini
      data_format: summary
      mean_pd: avg_probability
      defaults: default_count
      volume: total_count
      metric_type: gini
    dataset: grade_summary

Portfolio Risk Analysis

datasets:
  portfolio_data:
    type: csv
    source: data/portfolio_performance.csv
  exposure_data:
    type: csv
    source: data/exposure_predictions.csv
  regional_data:
    type: csv
    source: data/regional_exposures.csv
collections:
  product_accuracy:
    metrics:
    - name:
      - mortgage_accuracy
      - auto_accuracy
      - personal_accuracy
      segment:
      - - mortgage_2023
      - - auto_2023
      - - personal_2023
      prob_def: pd_estimate
      default: default_12m
      metric_type: default_accuracy
      data_format: record
    dataset: portfolio_data
  ead_scenarios:
    metrics:
    - name:
      - ead_optimistic
      - ead_baseline
      - ead_conservative
      segment:
      - - scenario_opt
      - - scenario_base
      - - scenario_cons
      predicted_ead: ead_prediction
      actual_ead: actual_ead
      default: default_flag
      metric_type: ead_accuracy
      data_format: record
    dataset: exposure_data
  regional_exposure_means:
    metrics:
    - name:
      - northeast_mean
      - southeast_mean
      - midwest_mean
      - west_mean
      - southwest_mean
      segment:
      - - northeast
      - - southeast
      - - midwest
      - - west
      - - southwest
      variable: exposure_amount
      metric_type: mean
      data_format: record
    dataset: regional_data
  regional_exposure_medians:
    metrics:
    - name:
      - northeast_median
      - southeast_median
      - midwest_median
      - west_median
      - southwest_median
      segment:
      - - northeast
      - - southeast
      - - midwest
      - - west
      - - southwest
      variable: exposure_amount
      metric_type: median
      data_format: record
    dataset: regional_data

Complex Multi-dimensional Analysis

datasets:
  performance_data:
    type: csv
    source: data/quarterly_performance.csv
  calibration_sample:
    type: csv
    source: data/calibration_test.csv
  customer_analytics:
    type: csv
    source: data/customer_analysis.csv
collections:
  multi_dimensional_accuracy:
    metrics:
    - name:
      - prime_q1_acc
      - prime_q2_acc
      - subprime_q1_acc
      - subprime_q2_acc
      segment:
      - - prime_customers
        - q1
      - - prime_customers
        - q2
      - - subprime_customers
        - q1
      - - subprime_customers
        - q2
      prob_def: default_prediction
      default: default_outcome
      metric_type: default_accuracy
      data_format: record
    dataset: performance_data
  comprehensive_calibration:
    metrics:
    - name:
      - hl_overall
      - hl_prime
      - hl_subprime
      segment:
      - null
      - - prime_customers
      - - subprime_customers
      prob_def: calibrated_probability
      default: default_indicator
      bands: 10
      metric_type: hosmer_lemeshow
      data_format: record
    dataset: calibration_sample
  multi_variable_summaries:
    metrics:
    - name:
      - avg_income_prime
      - avg_income_sub
      - avg_score_prime
      - avg_score_sub
      variable: annual_income
      segment:
      - - prime_segment
      - - subprime_segment
      - - prime_segment
      - - subprime_segment
      metric_type: mean
      data_format: record
    dataset: customer_analytics

Production Monitoring Recipe

datasets:
  daily_predictions:
    type: csv
    source: data/daily_model_output.csv
  current_week_scores:
    type: csv
    source: data/current_week.csv
  baseline_week_scores:
    type: csv
    source: data/baseline_week.csv
  monthly_sample:
    type: csv
    source: data/monthly_calibration.csv
  current_portfolio:
    type: csv
    source: data/current_exposures.csv
  origination_data:
    type: csv
    source: data/origination_channels.csv
collections:
  daily_performance:
    metrics:
    - name:
      - daily_model_accuracy
      prob_def: model_output
      default: observed_outcome
      metric_type: default_accuracy
      data_format: record
    dataset: daily_predictions
  weekly_drift:
    metrics:
    - name:
      - weekly_score_drift
      variable: model_score
      metric_type: jeffreys_test
      data_format: record
    dataset: current_week_scores
  monthly_calibration:
    metrics:
    - name:
      - monthly_calibration
      prob_def: probability_estimate
      default: default_outcome
      bands: 10
      metric_type: hosmer_lemeshow
      data_format: record
    dataset: monthly_sample
  exposure_monitoring_mean:
    metrics:
    - name:
      - current_avg_exposure
      variable: exposure_at_default
      metric_type: mean
      data_format: record
    dataset: current_portfolio
  exposure_monitoring_median:
    metrics:
    - name:
      - current_med_exposure
      variable: exposure_at_default
      metric_type: median
      data_format: record
    dataset: current_portfolio
  channel_performance:
    metrics:
    - name:
      - online_auc
      - branch_auc
      - broker_auc
      - partner_auc
      segment:
      - - online
      - - branch
      - - broker
      - - partner
      prob_def: approval_score
      default: early_default
      metric_type: auc
      data_format: record
    dataset: origination_data

Fan-out Validation Examples

Correct Fan-out Structure

datasets:
  validation_data:
    type: csv
    source: data/multi_model_validation.csv
  portfolio_data:
    type: csv
    source: data/portfolio.csv
collections:
  valid_fanout:
    metrics:
    - name:
      - metric_a
      - metric_b
      - metric_c
      segment:
      - - seg_a
      - - seg_b
      - - seg_c
      prob_def: predicted_default
      default: actual_outcome
      metric_type: default_accuracy
      data_format: record
    dataset: validation_data
  segment_analysis:
    metrics:
    - name:
      - retail_mean
      - corporate_mean
      - sme_mean
      variable: exposure_amount
      segment:
      - - retail
      - - corporate
      - - sme
      metric_type: mean
      data_format: record
    dataset: portfolio_data

Advanced Fan-out Patterns

datasets:
  ead_validation:
    type: csv
    source: data/ead_test_set.csv
  validation_data:
    type: csv
    source: data/model_validation.csv
collections:
  ead_matrix:
    metrics:
    - name:
      - conservative_95
      - aggressive_99
      - baseline_90
      - stress_95
      segment:
      - - model1
      - - model1
      - - model2
      - - model2
      predicted_ead: ead_prediction
      actual_ead: actual_ead
      default: default_flag
      metric_type: ead_accuracy
      data_format: record
    dataset: ead_validation
  model_segment_matrix:
    metrics:
    - name:
      - v1_prime
      - v1_subprime
      - v2_prime
      - v2_subprime
      - v3_prime
      - v3_subprime
      segment:
      - - prime
      - - subprime
      - - prime
      - - subprime
      - - prime
      - - subprime
      prob_def: model_score
      default: default_flag
      metric_type: auc
      data_format: record
    dataset: validation_data

Notes

  • YAML Structure: Metrics live under collections.<name>.metrics as list items
  • Dataset References: dataset can be set at the collection level (default) or overridden per metric
  • Fan-out Fields: Only name and segment support fan-out expansion (must be lists with matching lengths)
  • Non-Fan-out Fields: Fields like dataset, prob_def, default, predicted_ead, actual_ead, variable, bands are single values
  • Segment Format: Use segment: ["column_name"] for single grouping, or fan-out with lists of segments
  • Length Matching: When using fan-out, name and segment lists must have exactly the same length
  • Naming: Each expanded metric gets a unique name from the fan-out expansion
  • Validation: The recipe validator checks for proper list length matching and required fields