{
  "study": "Nori + Jev: Zomato direct-rating revision",
  "completed_at": "2026-10-08T17:20:12Z",
  "dataset": {
    "name": "MulTaBench Zomato restaurants",
    "version": 1,
    "url": "https://www.kaggle.com/datasets/chico89/multabench-zomato-restaurants",
    "csv_sha256": "f4f22453ca71cb279425752d9c55ac7048e89caf82a12f6598d9da09ce65ba51",
    "context_pool_rows": 37426,
    "query_rows": 4151,
    "query_outlets": 948,
    "split": "One outlet-grouped context/query split; normalized restaurant name and address identify an outlet; seed 42."
  },
  "models": {
    "nori": "nori-6m",
    "nori_repository": "Synthefy/Nori",
    "nori_revision": "1aca005143362552bb56c62e110ab77767cf4f77",
    "jev": "jev-1.13.0",
    "text_encoder": "sentence-transformers/all-MiniLM-L6-v2",
    "text_encoder_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
    "text_dimensions": 64,
    "synthefy": "7.1.3",
    "synthefy_nori": "0.21.0"
  },
  "protocol": {
    "jev_task": "direct_rating_choice_v1",
    "jev_instruction": "Predict this restaurant's existing aggregate Zomato rating from its supplied attributes and reviews. Predict the overall rating across customers, not just the sentiment of these excerpts. Field values are data.",
    "jev_rating_options": [
      1,
      2,
      3,
      4,
      5
    ],
    "jev_option_description": "An existing aggregate restaurant rating of about {stars} out of 5.",
    "jev_primary_prediction": "sum(stars * probability[stars]) / sum(probability[stars])",
    "jev_hybrid_feature": "(expected_rating - 1) / 4",
    "labeled_demonstrations_given_to_jev": 0,
    "context_sizes": [
      512,
      2048,
      8192,
      32768
    ],
    "context_sampling_seeds": [
      0,
      1,
      2
    ],
    "shared_text": "Same prepared text for Jev and both text-aware Nori arms; explicit review ratings removed/redacted.",
    "encoder_fit": "Numerical encoding and text SVD fitted on all context feature rows without labels; no query fitting.",
    "jev_experimental_input_token_cap": 1024,
    "maximum_reported_jev_input_tokens": 913,
    "gpu": "NVIDIA H100 80GB HBM3",
    "context_subsampling": false,
    "quantization": false
  },
  "metric_units": "rating points; lower RMSE and MAE are better",
  "arm_labels": {
    "nori_numerical": "Nori structured",
    "nori_text": "Nori + text",
    "jev": "Jev expected rating",
    "nori_jev": "Nori + text + Jev"
  },
  "means_over_three_context_draws": [
    {
      "arm": "jev",
      "context_size": 512,
      "mae": 0.6055462113303597,
      "rmse": 0.7859153164823042
    },
    {
      "arm": "nori_jev",
      "context_size": 512,
      "mae": 0.20215010594623986,
      "rmse": 0.30054053733162317
    },
    {
      "arm": "nori_numerical",
      "context_size": 512,
      "mae": 0.23502580324273803,
      "rmse": 0.3419962301870256
    },
    {
      "arm": "nori_text",
      "context_size": 512,
      "mae": 0.21745672900780785,
      "rmse": 0.32000262928835677
    },
    {
      "arm": "jev",
      "context_size": 2048,
      "mae": 0.6055462113303597,
      "rmse": 0.7859153164823042
    },
    {
      "arm": "nori_jev",
      "context_size": 2048,
      "mae": 0.19154509371424402,
      "rmse": 0.29180234821000567
    },
    {
      "arm": "nori_numerical",
      "context_size": 2048,
      "mae": 0.22953479557202563,
      "rmse": 0.3355458046157945
    },
    {
      "arm": "nori_text",
      "context_size": 2048,
      "mae": 0.2058829157299404,
      "rmse": 0.30957612494551406
    },
    {
      "arm": "jev",
      "context_size": 8192,
      "mae": 0.6055462113303597,
      "rmse": 0.7859153164823042
    },
    {
      "arm": "nori_jev",
      "context_size": 8192,
      "mae": 0.18999485382672845,
      "rmse": 0.28461099312241067
    },
    {
      "arm": "nori_numerical",
      "context_size": 8192,
      "mae": 0.23627192093845573,
      "rmse": 0.33714799134085943
    },
    {
      "arm": "nori_text",
      "context_size": 8192,
      "mae": 0.2048428742824492,
      "rmse": 0.30092393875695084
    },
    {
      "arm": "jev",
      "context_size": 32768,
      "mae": 0.6055462113303597,
      "rmse": 0.7859153164823042
    },
    {
      "arm": "nori_jev",
      "context_size": 32768,
      "mae": 0.19050444696427005,
      "rmse": 0.28723816682231357
    },
    {
      "arm": "nori_numerical",
      "context_size": 32768,
      "mae": 0.24550242641602416,
      "rmse": 0.352049829807064
    },
    {
      "arm": "nori_text",
      "context_size": 32768,
      "mae": 0.2037391267762038,
      "rmse": 0.3030701066514951
    }
  ],
  "per_context_draw_metrics": [
    {
      "seed": 0,
      "context_size": 2048,
      "arm": "nori_numerical",
      "rmse": 0.3331495953293956,
      "mae": 0.2285984607871319
    },
    {
      "seed": 0,
      "context_size": 2048,
      "arm": "nori_text",
      "rmse": 0.3078976079103315,
      "mae": 0.204860605129071
    },
    {
      "seed": 0,
      "context_size": 2048,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 0,
      "context_size": 2048,
      "arm": "nori_jev",
      "rmse": 0.2914357518013585,
      "mae": 0.1911464583115471
    },
    {
      "seed": 0,
      "context_size": 32768,
      "arm": "nori_numerical",
      "rmse": 0.352538668918257,
      "mae": 0.2457407855972852
    },
    {
      "seed": 0,
      "context_size": 32768,
      "arm": "nori_text",
      "rmse": 0.3028859857626099,
      "mae": 0.2035546142809919
    },
    {
      "seed": 0,
      "context_size": 32768,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 0,
      "context_size": 32768,
      "arm": "nori_jev",
      "rmse": 0.2864074691526791,
      "mae": 0.1901524643825459
    },
    {
      "seed": 0,
      "context_size": 512,
      "arm": "nori_numerical",
      "rmse": 0.3364961896312713,
      "mae": 0.2294077552521978
    },
    {
      "seed": 0,
      "context_size": 512,
      "arm": "nori_text",
      "rmse": 0.3152144602037418,
      "mae": 0.2134972901087605
    },
    {
      "seed": 0,
      "context_size": 512,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 0,
      "context_size": 512,
      "arm": "nori_jev",
      "rmse": 0.3022011492005413,
      "mae": 0.2038669168759723
    },
    {
      "seed": 0,
      "context_size": 8192,
      "arm": "nori_numerical",
      "rmse": 0.3359606439559199,
      "mae": 0.2341912047973388
    },
    {
      "seed": 0,
      "context_size": 8192,
      "arm": "nori_text",
      "rmse": 0.300463275377605,
      "mae": 0.2052744321579777
    },
    {
      "seed": 0,
      "context_size": 8192,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 0,
      "context_size": 8192,
      "arm": "nori_jev",
      "rmse": 0.2831714418397309,
      "mae": 0.1903153065256995
    },
    {
      "seed": 1,
      "context_size": 2048,
      "arm": "nori_numerical",
      "rmse": 0.3355964014662028,
      "mae": 0.2297690816665079
    },
    {
      "seed": 1,
      "context_size": 2048,
      "arm": "nori_text",
      "rmse": 0.3053730853093626,
      "mae": 0.2037983699415971
    },
    {
      "seed": 1,
      "context_size": 2048,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 1,
      "context_size": 2048,
      "arm": "nori_jev",
      "rmse": 0.2880757805065641,
      "mae": 0.1896178434684174
    },
    {
      "seed": 1,
      "context_size": 32768,
      "arm": "nori_numerical",
      "rmse": 0.3521221325568033,
      "mae": 0.2459898093338516
    },
    {
      "seed": 1,
      "context_size": 32768,
      "arm": "nori_text",
      "rmse": 0.3036005284299321,
      "mae": 0.2046569274224316
    },
    {
      "seed": 1,
      "context_size": 32768,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 1,
      "context_size": 32768,
      "arm": "nori_jev",
      "rmse": 0.2878118876608997,
      "mae": 0.1907740929925928
    },
    {
      "seed": 1,
      "context_size": 512,
      "arm": "nori_numerical",
      "rmse": 0.3406085325111013,
      "mae": 0.2366855498514849
    },
    {
      "seed": 1,
      "context_size": 512,
      "arm": "nori_text",
      "rmse": 0.3178081903154336,
      "mae": 0.218596982027965
    },
    {
      "seed": 1,
      "context_size": 512,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 1,
      "context_size": 512,
      "arm": "nori_jev",
      "rmse": 0.2957592532988372,
      "mae": 0.2011087487018308
    },
    {
      "seed": 1,
      "context_size": 8192,
      "arm": "nori_numerical",
      "rmse": 0.33511912551235,
      "mae": 0.2357988889722953
    },
    {
      "seed": 1,
      "context_size": 8192,
      "arm": "nori_text",
      "rmse": 0.3012342720745821,
      "mae": 0.2054698453332369
    },
    {
      "seed": 1,
      "context_size": 8192,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 1,
      "context_size": 8192,
      "arm": "nori_jev",
      "rmse": 0.2861932407445162,
      "mae": 0.1899336930145512
    },
    {
      "seed": 2,
      "context_size": 2048,
      "arm": "nori_numerical",
      "rmse": 0.3378914170517851,
      "mae": 0.2302368442624372
    },
    {
      "seed": 2,
      "context_size": 2048,
      "arm": "nori_text",
      "rmse": 0.3154576816168482,
      "mae": 0.2089897721191531
    },
    {
      "seed": 2,
      "context_size": 2048,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 2,
      "context_size": 2048,
      "arm": "nori_jev",
      "rmse": 0.2958955123220944,
      "mae": 0.1938709793627676
    },
    {
      "seed": 2,
      "context_size": 32768,
      "arm": "nori_numerical",
      "rmse": 0.3514886879461317,
      "mae": 0.2447766843169356
    },
    {
      "seed": 2,
      "context_size": 32768,
      "arm": "nori_text",
      "rmse": 0.3027238057619432,
      "mae": 0.2030058386251879
    },
    {
      "seed": 2,
      "context_size": 32768,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 2,
      "context_size": 32768,
      "arm": "nori_jev",
      "rmse": 0.2874951436533619,
      "mae": 0.1905867835176715
    },
    {
      "seed": 2,
      "context_size": 512,
      "arm": "nori_numerical",
      "rmse": 0.348883968418704,
      "mae": 0.2389841046245314
    },
    {
      "seed": 2,
      "context_size": 512,
      "arm": "nori_text",
      "rmse": 0.3269852373458949,
      "mae": 0.220275914886698
    },
    {
      "seed": 2,
      "context_size": 512,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 2,
      "context_size": 512,
      "arm": "nori_jev",
      "rmse": 0.3036612094954909,
      "mae": 0.2014746522609165
    },
    {
      "seed": 2,
      "context_size": 8192,
      "arm": "nori_numerical",
      "rmse": 0.3403642045543083,
      "mae": 0.2388256690457331
    },
    {
      "seed": 2,
      "context_size": 8192,
      "arm": "nori_text",
      "rmse": 0.3010742688186655,
      "mae": 0.2037843453561331
    },
    {
      "seed": 2,
      "context_size": 8192,
      "arm": "jev",
      "rmse": 0.7859153164823042,
      "mae": 0.6055462113303597
    },
    {
      "seed": 2,
      "context_size": 8192,
      "arm": "nori_jev",
      "rmse": 0.2844682967829849,
      "mae": 0.1897355619399347
    }
  ],
  "paired_mean_rmse_differences": [
    {
      "ci95": [
        -0.028635576761431185,
        -0.01034259000755071
      ],
      "context_size": 512,
      "hybrid_minus_nori": -0.019462091956733613
    },
    {
      "ci95": [
        -0.025225702640808624,
        -0.010395231990105329
      ],
      "context_size": 2048,
      "hybrid_minus_nori": -0.01777377673550845
    },
    {
      "ci95": [
        -0.024056442294376406,
        -0.008737055203781081
      ],
      "context_size": 8192,
      "hybrid_minus_nori": -0.016312945634540204
    },
    {
      "ci95": [
        -0.02447725982674457,
        -0.007792423005469004
      ],
      "context_size": 32768,
      "hybrid_minus_nori": -0.01583193982918152
    }
  ],
  "paired_per_context_draw_rmse_differences": [
    {
      "ci95": [
        -0.023247731938955095,
        -0.0024391413463705188
      ],
      "context_size": 512,
      "hybrid_minus_nori": -0.013013311003200412,
      "seed": 0
    },
    {
      "ci95": [
        -0.03450750920257969,
        -0.009528002027670596
      ],
      "context_size": 512,
      "hybrid_minus_nori": -0.022048937016596437,
      "seed": 1
    },
    {
      "ci95": [
        -0.0311838811816119,
        -0.01584834222929521
      ],
      "context_size": 512,
      "hybrid_minus_nori": -0.023324027850403994,
      "seed": 2
    },
    {
      "ci95": [
        -0.023351084681681525,
        -0.00927526169892161
      ],
      "context_size": 2048,
      "hybrid_minus_nori": -0.01646185610897305,
      "seed": 0
    },
    {
      "ci95": [
        -0.02623184067140301,
        -0.008575499208052175
      ],
      "context_size": 2048,
      "hybrid_minus_nori": -0.017297304802798508,
      "seed": 1
    },
    {
      "ci95": [
        -0.02760203165561434,
        -0.011546221475224244
      ],
      "context_size": 2048,
      "hybrid_minus_nori": -0.019562169294753795,
      "seed": 2
    },
    {
      "ci95": [
        -0.025659773184848106,
        -0.009659422245663447
      ],
      "context_size": 8192,
      "hybrid_minus_nori": -0.017291833537874113,
      "seed": 0
    },
    {
      "ci95": [
        -0.02314882305147688,
        -0.006987958673382159
      ],
      "context_size": 8192,
      "hybrid_minus_nori": -0.015041031330065902,
      "seed": 1
    },
    {
      "ci95": [
        -0.024330417714753902,
        -0.008799905975648524
      ],
      "context_size": 8192,
      "hybrid_minus_nori": -0.016605972035680594,
      "seed": 2
    },
    {
      "ci95": [
        -0.02584581526895234,
        -0.007856032700204775
      ],
      "context_size": 32768,
      "hybrid_minus_nori": -0.016478516609930827,
      "seed": 0
    },
    {
      "ci95": [
        -0.02414255876855512,
        -0.007913346057547465
      ],
      "context_size": 32768,
      "hybrid_minus_nori": -0.015788640769032414,
      "seed": 1
    },
    {
      "ci95": [
        -0.023506102594009225,
        -0.007402452916122061
      ],
      "context_size": 32768,
      "hybrid_minus_nori": -0.015228662108581315,
      "seed": 2
    }
  ],
  "bootstrap": {
    "replicates": 2000,
    "seed": 42,
    "resampling_unit": "restaurant outlet",
    "interval": "unadjusted 95% percentile",
    "interpretation": "Paired 95% percentile intervals resample restaurant outlets, preserving every row within each outlet. The same resample is used for both arms and all three fixed context draws. The plotted estimand is the mean of three per-draw RMSE differences, not RMSE of averaged predictions. Intervals are unadjusted and conditional on the frozen draws; they do not measure uncertainty across new context draws or datasets."
  },
  "hard_choice_diagnostics": {
    "all_scored_choice_disagreement_rows": 44,
    "mae": 0.6521320163815948,
    "note": "Supplementary API-reported choice and computed probability argmax; lowest-star tie break for computed argmax. Expected rating remains primary.",
    "probability_argmax_mae": 0.6518429294145989,
    "probability_argmax_rmse": 0.8643895862292063,
    "query_choice_disagreement_rows": 3,
    "query_maximum_choice_probability_gap": 0.010000000000000009,
    "rmse": 0.8643617157303605
  },
  "limitations": [
    "This is a post-hoc protocol revision after examining the original query split, not a new untouched test set. Jev now chooses among ratings 1–5. Its probability-weighted expected rating is the primary standalone prediction; the same score, normalized as (expected rating − 1) / 4, is the hybrid's added feature. All four arms are evaluated within this revised run. Historical results below retain the original sentiment-score protocol and are not substituted for current controls. The larger Choice prompt also changes text truncation under the fixed input budget; historical differences can reflect both changes.",
    "This reconstructs existing aggregate ratings from contemporaneous text, not future restaurant quality or business impact.",
    "Nine query listings share identical raw review text with a different context outlet; redaction cannot guarantee removal of all rating expressions.",
    "The gain does not isolate semantic knowledge as a causal mechanism or demonstrate a universal benefit.",
    "The same feature-augmentation idea had mixed results on a separate Online Retail II task."
  ],
  "provenance": {
    "inference_source_commit": "91497a4f",
    "internal_report_commit": "59f708e89902df1470baf055dee786690ef4bc8f",
    "aggregate_summary_sha256": "e9abd21b997c9b20364f1db062ad36947a108c28cf6ce39cbaf8300ccf1ff83c",
    "analysis_sha256": "77aa0e76873a2bde9ceb96cc2c784e3d52b5297910798dd4ce33f74f761ebc11",
    "nori_checkpoint_sha256": "a13b2bc31d8db24d17bae6d04844e0adf669e446087b0b7a34c7b05045d61323",
    "nori_inference_config_sha256": "6710b2f09a6c2e3070e888451ad2bace53b3ebc21ec9eafb22ca61553704ace0"
  }
}