{ "version": "v3.2", "generated_utc": "19 September 2026", "track": "best", "other_track": "constrained", "methodology": { "index_name": "Recommender Performance Index", "index_short": "RPI", "value_index_name": "Recommender Value Index", "value_index_short": "RVI", "value_index_note": "quality per dollar, expressed as a ratio to ItemKNN, which therefore reads exactly 100 here by definition. Quality-per-dollar has no natural unit and a bare ratio is unreadable, so the scale borrows one — this is a declared unit, not a statement about where ItemKNN places. Its position on the quality index is measured like every other system's.", "version": "v3.1", "frozen_at": "2026-08-22T00:00:00Z", "fingerprint": "52706396658de50c", "calibration": { "state": "frozen", "fingerprint": "05eb9f3f645e581b", "minted_at": "2026-08-23T13:27:52Z", "n_constants": 167, "note": "the field for each metric and dataset is a fixed list of numbers published in configs/v3_calibration.json, so every system including the two reference systems is scored against a distribution that does not move when the leaderboard grows" }, "normalisation": "frozen-field percentile: a system's score on a metric is where its measurement falls in the distribution of every entrant's measurement on that metric and dataset, pooled across the calibrating release's runs and then frozen. Nothing is anchored to a chosen system, so the scale cannot invert or explode, and no system is pinned to either end. Freezing the field is what keeps adding an entrant from moving anyone else's score.", "normalisation_id": "frozen_field_percentile", "supersedes": "v3.0 used a two-point anchored scale dividing by (itemknn - popularity). It was withdrawn: on two of ten datasets that difference was negative, inverting the scale, and where it was small a worse result could reach -381 before being floored. Scores are not comparable across the change.", "clip": [ 0, 100 ], "saturation": "a percentile is bounded; a system beyond either end of the frozen field saturates at 0 or 100 on that metric, and how often that happens is reported", "dimension_weights": { "relevance": 0.35, "cold_start": 0.15, "long_tail": 0.15, "beyond_accuracy": 0.1, "multi_objective": 0.1, "robustness": 0.15 }, "consistency": "reported as a separate `spread` column (cross-dataset standard deviation of the relevance dimension), not scored: how much to penalise a system that wins on one dataset and loses on another is a judgement about what the reader wants, not a measurement", "dimension_metrics": { "relevance": { "ndcg@10": { "weight": 0.45, "higher_is_better": true }, "recall@20": { "weight": 0.3, "higher_is_better": true }, "mrr@10": { "weight": 0.15, "higher_is_better": true }, "hitrate@10": { "weight": 0.1, "higher_is_better": true } }, "cold_start": { "cohort:cold_users_1_4:ndcg@10": { "weight": 0.45, "higher_is_better": true }, "cohort:cold_items:ndcg@10": { "weight": 0.35, "higher_is_better": true }, "cohort:new_items_since_train:ndcg@10": { "weight": 0.2, "higher_is_better": true } }, "long_tail": { "cohort:tail_target_items:ndcg@10": { "weight": 0.4, "higher_is_better": true }, "tail_share@20": { "weight": 0.3, "higher_is_better": true }, "coverage@20": { "weight": 0.3, "higher_is_better": true } }, "beyond_accuracy": { "novelty@20": { "weight": 0.35, "higher_is_better": true }, "ild@20": { "weight": 0.35, "higher_is_better": true }, "popularity_percentile@20": { "weight": 0.15, "higher_is_better": false }, "provider_gini": { "weight": 0.15, "higher_is_better": false } }, "multi_objective": { "mo_weighted_utility": { "weight": 0.65, "higher_is_better": true }, "mo_negative_signal_rate": { "weight": 0.35, "higher_is_better": false } }, "robustness": { "cohort:sparse_history:ndcg@10": { "weight": 0.35, "higher_is_better": true }, "cohort:dense_history:ndcg@10": { "weight": 0.25, "higher_is_better": true }, "cohort:heavy_users_20plus:ndcg@10": { "weight": 0.2, "higher_is_better": true }, "cohort:warm_users_5_19:ndcg@10": { "weight": 0.2, "higher_is_better": true } } }, "aggregation": "mean percentile across datasets per metric, then weighted mean within each dimension, then weighted mean across dimensions; dimensions with no measurable metric are dropped and the remaining weights renormalised", "quality_only": "cost, latency, model size and retraining burden are NOT inputs to this index; they are reported separately and combined only in the Recommender Value Index" }, "reference_deployment": { "active_users": 1000000, "slates_per_user_per_month": 10, "retrains_per_month": 4.35, "serving_utilisation": 0.35, "slates_per_month": 10000000, "note": "stated assumption, not a measurement; the per-slate and per-retrain figures it multiplies ARE measured" }, "uncalibrated_datasets": [], "leaderboard": [ { "rank": 1, "key": "itemknn_multitask", "system": "ItemKNN + Multi-task ranker", "index": 68.82, "family": "multi-stage", "generative": false, "confidence": "reproduction", "origin": "Multi-task ranking over several engagement signals", "citation": "Ma et al., 'Modeling Task Relationships in Multi-task Learning with Multi-gate Mixture-of-Experts', KDD 2018; Zhao et al., 'Recommending What Video to Watch Next', RecSys 2019.", "summary": "A neighbourhood model produces the candidates, and the ranking stage carries one output head per logged engagement signal — click, long view, like, share, follow, and the negative signals — combined into a single order by weights fixed before any result was measured (Ma et al., KDD 2018). On a dataset that logs only one signal there is only one head, and it is reported as a pointwise ranker there.", "stages": "retrieval + multi-task ranking", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": "itemknn", "retriever_label": null, "ranker_label": "Multi-task ranker", "is_new": false, "dimensions": { "relevance": 78.99, "cold_start": 63.93, "long_tail": 50.26, "beyond_accuracy": 53.11, "multi_objective": 69.64, "robustness": 78.46 }, "per_dataset_relevance": { "amazon-beauty-2023": 74.34782608695653, "goodreads-poetry": 82.60869565217392, "hm-fashion": 81.9047619047619, "kuairand-pure": 85.54347826086956, "kuairec": 86.52173913043478, "lastfm-1k": 86.42857142857143, "mind-small": 66.5217391304348, "ml-1m": 85.10869565217392, "ml-32m": 71.41304347826087, "retailrocket": 69.45652173913044 }, "n_datasets": 10, "spread": 7.343986192769808, "monthly_cost_usd": 64.24727830912043, "train_cost_usd": 0.005819726882651014, "inference_cost_per_1k": 0.00642219624971809, "latency_p50_ms": 3.4506985808711894, "latency_p95_ms": 3.4506985808711894, "model_mb": 42.967064400000005, "requires_gpu": true, "retrain_note": "retriever refit on train+valid for serving; ranker weights carried over from the valid-window fit", "latency_caveat": "includes constructing the ranker's features online, in Python, for every request — work a deployed pipeline precomputes or caches, so this is an upper bound", "value_index": 7.712291872701536, "other_track_index": 69, "win_rate": 68.8076691241284, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.011234498400048335, "recall@20": 0.02183588542540791, "mrr@10": 0.014565674603174603, "hitrate@10": 0.03375, "coverage@20": 0.2938157557095201, "novelty@20": 11.222645182579052, "ild@20": 1, "tail_share@20": 0.0658125 }, "goodreads-poetry": { "ndcg@10": 0.07603617612829523, "recall@20": 0.15553488581382413, "mrr@10": 0.09615084532076719, "hitrate@10": 0.21809895833333334, "coverage@20": 0.44402342404409234, "novelty@20": 8.732104983474821, "ild@20": 0.454725191885965, "tail_share@20": 0.04814453125 }, "hm-fashion": { "ndcg@10": 0.008739844553529082, "recall@20": 0.017385963780959544, "mrr@10": 0.01785089285714286, "hitrate@10": 0.04825, "coverage@20": 0.19640311163360766, "novelty@20": 12.551132184202247, "ild@20": 0.6842766204140919, "tail_share@20": 0.0589875 }, "kuairand-pure": { "ndcg@10": 0.017043560978391437, "recall@20": 0.04089407021282732, "mrr@10": 0.027917361111111108, "hitrate@10": 0.07575, "coverage@20": 0.18158907016845735, "novelty@20": 9.33672361424518, "ild@20": 0.41396710526315805, "tail_share@20": 0.000025 }, "kuairec": { "ndcg@10": 0.06234622409351161, "recall@20": 0.0863490637289863, "mrr@10": 0.13117124676596043, "hitrate@10": 0.2854625550660793, "coverage@20": 0.040087863811092805, "novelty@20": 8.783307125845168, "ild@20": 0.6111087313156651, "tail_share@20": 0 }, "lastfm-1k": { "ndcg@10": 0.07124973334513691, "recall@20": 0.011513176668646949, "mrr@10": 0.1425996959015827, "hitrate@10": 0.3309143686502177, "coverage@20": 0.018012101880951262, "novelty@20": 10.85369923342856, "ild@20": 1, "tail_share@20": 0 }, "mind-small": { "ndcg@10": 0.12821163107495753, "recall@20": 0.27592724920525813, "mrr@10": 0.14378888888888888, "hitrate@10": 0.31875, "coverage@20": 0.15359741309620048, "novelty@20": 9.172188563499464, "ild@20": 0.9161236842105264, "tail_share@20": 0.0018125 }, "ml-1m": { "ndcg@10": 0.19666379900907097, "recall@20": 0.08888848384569599, "mrr@10": 0.34413496189925996, "hitrate@10": 0.6117850953206239, "coverage@20": 0.08704, "novelty@20": 8.526318588345577, "ild@20": 0.73215013131819, "tail_share@20": 0 }, "ml-32m": { "ndcg@10": 0.1034948361705291, "recall@20": 0.05875533970260243, "mrr@10": 0.19724164724164725, "hitrate@10": 0.40368722186904005, "coverage@20": 0.07187857961053837, "novelty@20": 9.623167997213887, "ild@20": 0.6715992956816087, "tail_share@20": 0 }, "retailrocket": { "ndcg@10": 0.019124202041535896, "recall@20": 0.03125, "mrr@10": 0.01867816091954023, "hitrate@10": 0.02586206896551724, "coverage@20": 0.22912801484230055, "novelty@20": 11.356173945056192, "ild@20": 1, "tail_share@20": 0.7375 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 1, "compared_on_n_datasets": 10 }, { "rank": 2, "key": "itemknn_mlp", "system": "ItemKNN + MLP ranker", "index": 66.89, "family": "multi-stage", "generative": false, "confidence": "reference", "origin": "Classical retrieval with a neural pointwise ranker", "citation": "Sarwar et al., WWW 2001 (retrieval); Covington, Adams & Sargin, RecSys 2016 (the deep-ranking stage this follows).", "summary": "A neighbourhood model produces a few hundred candidates and a multilayer perceptron reorders them, over features combining the retrieval score, item and user statistics, and content similarity. The retriever is fitted on the training window, the ranker is trained on candidates labelled from the validation window, and the retriever is refitted for serving; every part of that cost is charged to the system.", "stages": "retrieval + ranking", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": "itemknn", "retriever_label": null, "ranker_label": "MLP ranker", "is_new": false, "dimensions": { "relevance": 79.29, "cold_start": 58.84, "long_tail": 47.99, "beyond_accuracy": 51.87, "multi_objective": 65.4, "robustness": 75.91 }, "per_dataset_relevance": { "amazon-beauty-2023": 91.19565217391305, "goodreads-poetry": 80.43478260869566, "hm-fashion": 82.85714285714286, "kuairand-pure": 90.1086956521739, "kuairec": 98.69565217391305, "lastfm-1k": 81.42857142857143, "mind-small": 46.30434782608695, "ml-1m": 87.71739130434781, "ml-32m": 67.5, "retailrocket": 66.63043478260869 }, "n_datasets": 10, "spread": 14.533820485594564, "monthly_cost_usd": 63.554294600312076, "train_cost_usd": 0.007587411512949274, "inference_cost_per_1k": 0.0063521289360230744, "latency_p50_ms": 3.415221901294976, "latency_p95_ms": 3.415221901294976, "model_mb": 42.4633044, "requires_gpu": true, "retrain_note": "retriever refit on train+valid for serving; ranker weights carried over from the valid-window fit", "latency_caveat": "includes constructing the ranker's features online, in Python, for every request — work a deployed pipeline precomputes or caches, so this is an upper bound", "value_index": 7.5777541049620405, "other_track_index": 68.42, "win_rate": 66.38846385682146, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.012028371092285375, "recall@20": 0.02340403883276222, "mrr@10": 0.01562579365079365, "hitrate@10": 0.03475, "coverage@20": 0.264754939697203, "novelty@20": 11.125021292823776, "ild@20": 1, "tail_share@20": 0.03345 }, "goodreads-poetry": { "ndcg@10": 0.0741359912242087, "recall@20": 0.15666600889142282, "mrr@10": 0.09160207919973545, "hitrate@10": 0.21028645833333334, "coverage@20": 0.4691698243196693, "novelty@20": 8.655912830603144, "ild@20": 0.4445625171326755, "tail_share@20": 0.06422526041666667 }, "hm-fashion": { "ndcg@10": 0.008458184185904126, "recall@20": 0.018645186607059813, "mrr@10": 0.01769702380952381, "hitrate@10": 0.046, "coverage@20": 0.1905250256384602, "novelty@20": 12.445576234180793, "ild@20": 0.7037481025614089, "tail_share@20": 0.05655 }, "kuairand-pure": { "ndcg@10": 0.017741691083910972, "recall@20": 0.04025965000809188, "mrr@10": 0.027722817460317458, "hitrate@10": 0.078, "coverage@20": 0.18742538798249106, "novelty@20": 9.489251806818372, "ild@20": 0.4152763157894738, "tail_share@20": 0.00015 }, "kuairec": { "ndcg@10": 0.07363896005949103, "recall@20": 0.09145933976489072, "mrr@10": 0.15264771694287113, "hitrate@10": 0.3004405286343612, "coverage@20": 0.040087863811092805, "novelty@20": 8.587514392479042, "ild@20": 0.6582582737636172, "tail_share@20": 0 }, "lastfm-1k": { "ndcg@10": 0.07095760444205813, "recall@20": 0.00999821536517972, "mrr@10": 0.14216888981039927, "hitrate@10": 0.3309143686502177, "coverage@20": 0.017331957408884095, "novelty@20": 10.861110972193703, "ild@20": 1, "tail_share@20": 0 }, "mind-small": { "ndcg@10": 0.10650362550858743, "recall@20": 0.25373612634785236, "mrr@10": 0.11593541666666667, "hitrate@10": 0.2875, "coverage@20": 0.1438965238480194, "novelty@20": 9.309588894092922, "ild@20": 0.9062138157894738, "tail_share@20": 0.0018375 }, "ml-1m": { "ndcg@10": 0.19783511931226513, "recall@20": 0.0878644729050848, "mrr@10": 0.3501540535336029, "hitrate@10": 0.6143847487001733, "coverage@20": 0.08, "novelty@20": 8.49010823715874, "ild@20": 0.7405074997807475, "tail_share@20": 0 }, "ml-32m": { "ndcg@10": 0.10237952654632827, "recall@20": 0.05831562330105903, "mrr@10": 0.1955239205239205, "hitrate@10": 0.3954227590591227, "coverage@20": 0.07121038564337534, "novelty@20": 9.55149910852327, "ild@20": 0.6715918843439064, "tail_share@20": 0 }, "retailrocket": { "ndcg@10": 0.01698501161662412, "recall@20": 0.03358477011494253, "mrr@10": 0.01293103448275862, "hitrate@10": 0.034482758620689655, "coverage@20": 0.23794063079777367, "novelty@20": 11.421149910078615, "ild@20": 1, "tail_share@20": 0.7353448275862069 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 2, "compared_on_n_datasets": 10 }, { "rank": 3, "key": "vsknn", "system": "VS-KNN", "index": 65.17, "family": "neighbourhood", "generative": false, "confidence": "reference", "origin": "Vector-multiplication session-based kNN", "citation": "Ludewig & Jannach, 'Evaluation of Session-based Recommendation Algorithms', UMUAI 2018.", "summary": "Session-based nearest neighbours: it finds past sessions that resemble the one in progress, weighting the current session by position, and scores what those sessions went on to contain. It holds no long-term profile, so it produces recommendations for traffic with no identity attached to it.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 79.82, "cold_start": 60.36, "long_tail": 43.38, "beyond_accuracy": 39.77, "multi_objective": 59.04, "robustness": 78.57 }, "per_dataset_relevance": { "amazon-beauty-2023": 90.1086956521739, "goodreads-poetry": 72.6086956521739, "hm-fashion": 98.92857142857143, "kuairand-pure": 59.34782608695653, "kuairec": 60.97826086956522, "lastfm-1k": 85.53571428571428, "mind-small": 88.15217391304348, "ml-1m": 87.60869565217392, "ml-32m": 66.1413043478261, "retailrocket": 88.80434782608695 }, "n_datasets": 10, "spread": 13.14420861413183, "monthly_cost_usd": 34.51884978193563, "train_cost_usd": 0.000004316215990447113, "inference_cost_per_1k": 0.003451883100639607, "latency_p50_ms": 1.8823911636876747, "latency_p95_ms": 1.8823911636876747, "model_mb": 9.666758, "requires_gpu": false, "retrain_note": "index refresh only; no parameters to retrain", "latency_caveat": "", "value_index": 13.592505651495317, "other_track_index": 65.41, "win_rate": 64.2833959289685, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.011660521941527292, "recall@20": 0.023522751860642427, "mrr@10": 0.0159140873015873, "hitrate@10": 0.03425, "coverage@20": 0.5678727226071337, "novelty@20": 11.50589697776969, "ild@20": 1, "tail_share@20": 0.0609 }, "goodreads-poetry": { "ndcg@10": 0.07014321820636503, "recall@20": 0.1546667434984463, "mrr@10": 0.09112103174603176, "hitrate@10": 0.20442708333333334, "coverage@20": 0.26937650706166033, "novelty@20": 7.764559921199637, "ild@20": 0.46751858895285103, "tail_share@20": 0.00703125 }, "hm-fashion": { "ndcg@10": 0.011409717119931266, "recall@20": 0.019162814366005407, "mrr@10": 0.024352380952380955, "hitrate@10": 0.0615, "coverage@20": 0.14207458915930865, "novelty@20": 11.710438353796158, "ild@20": 0.7133346977408364, "tail_share@20": 0.0069875 }, "kuairand-pure": { "ndcg@10": 0.013847837012540538, "recall@20": 0.0370656797448646, "mrr@10": 0.023303472222222222, "hitrate@10": 0.0665, "coverage@20": 0.0708316752884998, "novelty@20": 8.597304794787032, "ild@20": 0.42475921052631593, "tail_share@20": 0.0000125 }, "kuairec": { "ndcg@10": 0.05652702143136791, "recall@20": 0.06878346239925104, "mrr@10": 0.10327075029718202, "hitrate@10": 0.18237885462555067, "coverage@20": 0.04942339373970346, "novelty@20": 7.907816578000828, "ild@20": 0.7884817809330199, "tail_share@20": 0 }, "lastfm-1k": { "ndcg@10": 0.07637357248857803, "recall@20": 0.00955641546725009, "mrr@10": 0.16629518280461678, "hitrate@10": 0.3265602322206096, "coverage@20": 0.013907781790890755, "novelty@20": 10.517045771606977, "ild@20": 1, "tail_share@20": 0.0005805515239477503 }, "mind-small": { "ndcg@10": 0.14556992934035518, "recall@20": 0.3130331955873074, "mrr@10": 0.16295525793650797, "hitrate@10": 0.354, "coverage@20": 0.267852330908111, "novelty@20": 8.522332750486447, "ild@20": 0.8988782894736843, "tail_share@20": 0.0041625 }, "ml-1m": { "ndcg@10": 0.19244163508043455, "recall@20": 0.09436781485865162, "mrr@10": 0.3321476713157822, "hitrate@10": 0.6126516464471404, "coverage@20": 0.12448, "novelty@20": 8.522457500055724, "ild@20": 0.7010853510331055, "tail_share@20": 0 }, "ml-32m": { "ndcg@10": 0.0987214117618008, "recall@20": 0.06020022599833975, "mrr@10": 0.194284755648392, "hitrate@10": 0.39732994278448824, "coverage@20": 0.03923253150057274, "novelty@20": 8.854963202226058, "ild@20": 0.7005032137375589, "tail_share@20": 0 }, "retailrocket": { "ndcg@10": 0.02570415854739308, "recall@20": 0.046515804597701146, "mrr@10": 0.025550766283524904, "hitrate@10": 0.06896551724137931, "coverage@20": 0.29359925788497215, "novelty@20": 10.690111140401655, "ild@20": 1, "tail_share@20": 0.3724137931034483 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 4, "compared_on_n_datasets": 8 }, { "rank": 4, "key": "ease", "system": "EASEᴿ", "index": 64.51, "family": "linear-autoencoder", "generative": false, "confidence": "reference", "origin": "Embarrassingly Shallow AutoEncoder", "citation": "Steck, 'Embarrassingly Shallow Autoencoders for Sparse Data', WWW 2019.", "summary": "A linear item-item autoencoder with a closed-form solution: one regularised Gram-matrix inverse, no iterations, and no hyperparameter beyond the ridge term (Steck, WWW 2019). Its memory and time are quadratic in catalogue size, so it is not attempted above roughly 25,000 items; the datasets where that applies are recorded as skipped rather than left out.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 75.65, "cold_start": 54.55, "long_tail": 58.49, "beyond_accuracy": 53.65, "multi_objective": 53.86, "robustness": 68.86 }, "per_dataset_relevance": { "amazon-beauty-2023": 60.54347826086956, "goodreads-poetry": 83.58695652173914, "kuairand-pure": 89.45652173913044, "kuairec": 37.934782608695656, "mind-small": 57.065217391304344, "ml-1m": 84.1304347826087, "ml-32m": 96.19565217391305, "retailrocket": 96.30434782608695 }, "n_datasets": 8, "spread": 19.906964018808214, "monthly_cost_usd": 4.759445457253691, "train_cost_usd": 0.004904727185693874, "inference_cost_per_1k": 0.00047381098939959233, "latency_p50_ms": 0.2545768344957571, "latency_p95_ms": 0.2545768344957571, "model_mb": 449.49777300000005, "requires_gpu": false, "retrain_note": "closed form; a full O(n_items^2) solve every time, no partial update", "latency_caveat": "", "value_index": 97.59528105289725, "other_track_index": 64.46, "win_rate": 65.12161252314583, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.009909636264594113, "recall@20": 0.01978934970959197, "mrr@10": 0.013446329365079364, "hitrate@10": 0.02825, "coverage@20": 0.6326020015396459, "novelty@20": 11.775226759182337, "ild@20": 1, "tail_share@20": 0.0740375 }, "goodreads-poetry": { "ndcg@10": 0.07349049382195426, "recall@20": 0.15922586892324744, "mrr@10": 0.09781668526785714, "hitrate@10": 0.21809895833333334, "coverage@20": 0.437822941784361, "novelty@20": 8.521401060280075, "ild@20": 0.465420264528509, "tail_share@20": 0.016341145833333334 }, "kuairand-pure": { "ndcg@10": 0.01704553328783983, "recall@20": 0.045462039763349324, "mrr@10": 0.025931448412698416, "hitrate@10": 0.07725, "coverage@20": 0.2034752619710837, "novelty@20": 9.50498328633385, "ild@20": 0.4260394736842107, "tail_share@20": 0.0003125 }, "kuairec": { "ndcg@10": 0.04696692282777333, "recall@20": 0.06262457972809642, "mrr@10": 0.08634675896790435, "hitrate@10": 0.16123348017621145, "coverage@20": 0.24107633168588688, "novelty@20": 8.242618821932988, "ild@20": 0.8016803424017199, "tail_share@20": 0.00013215859030837003 }, "mind-small": { "ndcg@10": 0.12076750185438781, "recall@20": 0.2611134842679251, "mrr@10": 0.13743501984126985, "hitrate@10": 0.3025, "coverage@20": 0.39558070600916195, "novelty@20": 8.820400545042117, "ild@20": 0.8948203947368423, "tail_share@20": 0.0064125 }, "ml-1m": { "ndcg@10": 0.17818179055618996, "recall@20": 0.1015250406967431, "mrr@10": 0.32363655745371517, "hitrate@10": 0.6152512998266898, "coverage@20": 0.27616, "novelty@20": 9.200336408046077, "ild@20": 0.6968006723978672, "tail_share@20": 0.00012998266897746967 }, "ml-32m": { "ndcg@10": 0.12429915027050366, "recall@20": 0.0753830320834233, "mrr@10": 0.23983844438389892, "hitrate@10": 0.45200254291163383, "coverage@20": 0.1399389079801451, "novelty@20": 9.878861351497845, "ild@20": 0.6838437083251651, "tail_share@20": 0.0001589319771137953 }, "retailrocket": { "ndcg@10": 0.028593002967237727, "recall@20": 0.046803160919540225, "mrr@10": 0.029392446633825946, "hitrate@10": 0.06896551724137931, "coverage@20": 0.29730983302411873, "novelty@20": 10.570993883766066, "ild@20": 1, "tail_share@20": 0.3267241379310345 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 2, "compared_on_n_datasets": 8 }, { "rank": 5, "key": "sasrec", "system": "SASRec", "index": 62.87, "family": "sequential", "generative": false, "confidence": "reproduction", "origin": "Self-attentive sequential recommendation", "citation": "Kang & McAuley, 'Self-Attentive Sequential Recommendation', ICDM 2018; trained with the full softmax of Klenitskiy & Vasilev, RecSys 2023.", "summary": "A causal transformer over the ordered item sequence (Kang & McAuley, ICDM 2018), trained with a softmax over the full catalogue rather than the original binary objective — the formulation Klenitskiy & Vasilev (RecSys 2023) found accounts for most of the reported gap to bidirectional models.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 60.32, "cold_start": 59.65, "long_tail": 67.53, "beyond_accuracy": 61.13, "multi_objective": 81.41, "robustness": 56.15 }, "per_dataset_relevance": { "amazon-beauty-2023": 51.52173913043478, "goodreads-poetry": 60.21739130434783, "hm-fashion": 32.023809523809526, "kuairand-pure": 99.67391304347827, "kuairec": 76.7391304347826, "lastfm-1k": 31.30952380952381, "mind-small": 84.67391304347825, "ml-1m": 11.41304347826087, "ml-32m": 90.97826086956522, "retailrocket": 64.67391304347825 }, "n_datasets": 10, "spread": 27.353078981328537, "monthly_cost_usd": 1.827146178908275, "train_cost_usd": 0.08601270511479499, "inference_cost_per_1k": 0.00014529909116589167, "latency_p50_ms": 0.08993730538481134, "latency_p95_ms": 0.12138541151242557, "model_mb": 8.266108, "requires_gpu": true, "retrain_note": "full retrain, or warm-start continuation from the last checkpoint", "latency_caveat": "", "value_index": 247.732038975163, "other_track_index": 35.1, "win_rate": 61.045810729358486, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.008009979999979405, "recall@20": 0.020000683897082205, "mrr@10": 0.010968948412698413, "hitrate@10": 0.0245, "coverage@20": 0.09892224788298691, "novelty@20": 10.348485582556037, "ild@20": 1, "tail_share@20": 0.0061625 }, "goodreads-poetry": { "ndcg@10": 0.06699482402456493, "recall@20": 0.14516018915076775, "mrr@10": 0.08888139674272487, "hitrate@10": 0.20768229166666666, "coverage@20": 0.7661040303134689, "novelty@20": 10.27539302502063, "ild@20": 0.49004677220394743, "tail_share@20": 0.16217447916666666 }, "hm-fashion": { "ndcg@10": 0.0035110235937032865, "recall@20": 0.007474580882924451, "mrr@10": 0.006751190476190477, "hitrate@10": 0.02075, "coverage@20": 0.23679931964281248, "novelty@20": 14.416300845603857, "ild@20": 0.7087203457100385, "tail_share@20": 0.1216375 }, "kuairand-pure": { "ndcg@10": 0.019280831768444688, "recall@20": 0.049585406638360266, "mrr@10": 0.027981646825396823, "hitrate@10": 0.086, "coverage@20": 0.18729274439580845, "novelty@20": 10.140510453726492, "ild@20": 0.4534078947368422, "tail_share@20": 0.0048 }, "kuairec": { "ndcg@10": 0.05776940855528339, "recall@20": 0.07991243732830416, "mrr@10": 0.12201769107055449, "hitrate@10": 0.28193832599118945, "coverage@20": 0.16090060406370127, "novelty@20": 9.939380266861553, "ild@20": 0.5588560517866897, "tail_share@20": 0.1130396475770925 }, "lastfm-1k": { "ndcg@10": 0.01826701108795635, "recall@20": 0.0023140455037801216, "mrr@10": 0.05048552076853963, "hitrate@10": 0.12772133526850507, "coverage@20": 0.07160279562831277, "novelty@20": 13.517524912954094, "ild@20": 1, "tail_share@20": 0.07895500725689406 }, "mind-small": { "ndcg@10": 0.13670571220013594, "recall@20": 0.3196037207675365, "mrr@10": 0.15174295634920634, "hitrate@10": 0.33925, "coverage@20": 0.22123416868768525, "novelty@20": 8.655786479625496, "ild@20": 0.9016901315789473, "tail_share@20": 0.0012625 }, "ml-1m": { "ndcg@10": 0.10174682483797279, "recall@20": 0.06289131726149079, "mrr@10": 0.20085692278066627, "hitrate@10": 0.44974003466204504, "coverage@20": 0.66624, "novelty@20": 10.787043801099745, "ild@20": 0.5808118187410038, "tail_share@20": 0.10125649913344888 }, "ml-32m": { "ndcg@10": 0.11329141971602254, "recall@20": 0.0738036451681601, "mrr@10": 0.23383687020050659, "hitrate@10": 0.4634456452638271, "coverage@20": 0.19797632684230623, "novelty@20": 12.272403915421116, "ild@20": 0.6304217792938768, "tail_share@20": 0.03343928798474253 }, "retailrocket": { "ndcg@10": 0.013681920690353264, "recall@20": 0.03160919540229885, "mrr@10": 0.013731527093596058, "hitrate@10": 0.034482758620689655, "coverage@20": 0.47309833024118736, "novelty@20": 11.02661018649399, "ild@20": 1, "tail_share@20": 0.5159482758620689 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 4, "compared_on_n_datasets": 10 }, { "rank": 6, "key": "vsknn_gbdt", "system": "VS-KNN + GBDT", "index": 60.76, "family": "multi-stage", "generative": false, "confidence": "reference", "origin": "Session-based neighbourhood retrieval with a boosted ranker", "citation": "Ludewig & Jannach, UMUAI 2018; Ke et al., NeurIPS 2017 (LightGBM).", "summary": "Session-based nearest neighbours produce the candidates and a pointwise gradient-boosted ranker orders them. The sequence signal enters through the retrieval stage rather than through a neural sequence model, which separates the contribution of sequence information from the contribution of the architecture that usually carries it.", "stages": "retrieval + ranking", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": "vsknn", "retriever_label": null, "ranker_label": "GBDT ranker", "is_new": false, "dimensions": { "relevance": 63.77, "cold_start": 59.33, "long_tail": 56.02, "beyond_accuracy": 50.89, "multi_objective": 70.58, "robustness": 59.91 }, "per_dataset_relevance": { "amazon-beauty-2023": 54.34782608695652, "goodreads-poetry": 70.65217391304348, "hm-fashion": 79.16666666666666, "kuairand-pure": 71.30434782608695, "kuairec": 42.71739130434783, "lastfm-1k": 68.57142857142857, "mind-small": 74.23913043478261, "ml-1m": 55.65217391304348, "ml-32m": 67.17391304347827, "retailrocket": 53.91304347826087 }, "n_datasets": 10, "spread": 10.864986576326556, "monthly_cost_usd": 94.24704668151557, "train_cost_usd": 0.005830853143494993, "inference_cost_per_1k": 0.009422168247034136, "latency_p50_ms": 5.06660523876047, "latency_p95_ms": 5.06660523876047, "model_mb": 17.930704000000002, "requires_gpu": false, "retrain_note": "retriever refit on train+valid for serving; ranker weights carried over from the valid-window fit", "latency_caveat": "includes constructing the ranker's features online, in Python, for every request — work a deployed pipeline precomputes or caches, so this is an upper bound", "value_index": 4.64146361375697, "other_track_index": 57.29, "win_rate": 59.012611860716355, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.00803095081958099, "recall@20": 0.02061338260147961, "mrr@10": 0.009920238095238095, "hitrate@10": 0.02775, "coverage@20": 0.16179112137541699, "novelty@20": 10.757657732140876, "ild@20": 1, "tail_share@20": 0.0128125 }, "goodreads-poetry": { "ndcg@10": 0.06654433780230559, "recall@20": 0.16452962528120083, "mrr@10": 0.08578016493055556, "hitrate@10": 0.21028645833333334, "coverage@20": 0.5811229762314847, "novelty@20": 9.090481401794255, "ild@20": 0.466932223135965, "tail_share@20": 0.06256510416666666 }, "hm-fashion": { "ndcg@10": 0.008108545596593946, "recall@20": 0.01740338171852679, "mrr@10": 0.016150396825396825, "hitrate@10": 0.048, "coverage@20": 0.07358863403286725, "novelty@20": 11.294570152765017, "ild@20": 0.7288232667407396, "tail_share@20": 0.0044125 }, "kuairand-pure": { "ndcg@10": 0.0152211829969402, "recall@20": 0.0426911712877942, "mrr@10": 0.021744642857142858, "hitrate@10": 0.07625, "coverage@20": 0.1853030905955697, "novelty@20": 9.455659376439995, "ild@20": 0.41411052631578965, "tail_share@20": 0.00055 }, "kuairec": { "ndcg@10": 0.03864000848925766, "recall@20": 0.07991824078869375, "mrr@10": 0.07230053842388645, "hitrate@10": 0.2185022026431718, "coverage@20": 0.1515650741350906, "novelty@20": 8.640773801684356, "ild@20": 0.6771442320661234, "tail_share@20": 0.00004405286343612335 }, "lastfm-1k": { "ndcg@10": 0.06779620650707699, "recall@20": 0.008838817832366987, "mrr@10": 0.14129172714078372, "hitrate@10": 0.32365747460087085, "coverage@20": 0.031427365261034755, "novelty@20": 11.146701379682167, "ild@20": 1, "tail_share@20": 0.0002177068214804064 }, "mind-small": { "ndcg@10": 0.12244048310172888, "recall@20": 0.32366925426869314, "mrr@10": 0.12156577380952382, "hitrate@10": 0.3615, "coverage@20": 0.15656157369981138, "novelty@20": 9.1187830255957, "ild@20": 0.9024190789473685, "tail_share@20": 0.0109375 }, "ml-1m": { "ndcg@10": 0.1634386714194863, "recall@20": 0.08147685826240038, "mrr@10": 0.28824929162884105, "hitrate@10": 0.5684575389948007, "coverage@20": 0.23456, "novelty@20": 8.79565209261766, "ild@20": 0.7070591153401795, "tail_share@20": 0 }, "ml-32m": { "ndcg@10": 0.0969056321428226, "recall@20": 0.06622318574402772, "mrr@10": 0.18925039606857788, "hitrate@10": 0.42085187539732993, "coverage@20": 0.10834287896143566, "novelty@20": 10.033391773379766, "ild@20": 0.6476717797844427, "tail_share@20": 0 }, "retailrocket": { "ndcg@10": 0.012062856237296502, "recall@20": 0.03153735632183908, "mrr@10": 0.010296934865900383, "hitrate@10": 0.034482758620689655, "coverage@20": 0.2792207792207792, "novelty@20": 11.443343832134687, "ild@20": 1, "tail_share@20": 0.7349137931034483 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 1, "compared_on_n_datasets": 10 }, { "rank": 7, "key": "recency_pop", "system": "Recency-Weighted Popularity", "index": 60.16, "family": "non-personalised", "generative": false, "confidence": "reference", "origin": "Standard non-personalised baseline with a recency kernel", "citation": "Ji et al., 'A Re-visit of the Popularity Baseline', SIGIR 2020.", "summary": "Catalogue-wide interaction count with an exponential recency kernel applied across the training window, so recently popular items outrank items that were popular earlier. Like plain popularity it returns the same order for every user, and it has nothing to train beyond the decay constant.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 76.58, "cold_start": 61.22, "long_tail": 23.39, "beyond_accuracy": 40.63, "multi_objective": 57.63, "robustness": 72.28 }, "per_dataset_relevance": { "amazon-beauty-2023": 73.3695652173913, "goodreads-poetry": 98.91304347826087, "hm-fashion": 65.71428571428572, "kuairand-pure": 34.02173913043479, "kuairec": 87.28260869565217, "lastfm-1k": 85, "mind-small": 98.91304347826087, "ml-1m": 79.13043478260869, "ml-32m": 94.13043478260869, "retailrocket": 49.347826086956516 }, "n_datasets": 10, "spread": 20.44177542050842, "monthly_cost_usd": 1.4505739484534947, "train_cost_usd": 0.00000902446435959107, "inference_cost_per_1k": 0.00014505346920335306, "latency_p50_ms": 0.09262549411522125, "latency_p95_ms": 0.09262549411522125, "model_mb": 0.1039488, "requires_gpu": false, "retrain_note": "counts with a decay kernel; recomputable in seconds", "latency_caveat": "", "value_index": 298.6294735374231, "other_track_index": 60.16, "win_rate": 59.40209579450336, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.011302085476693457, "recall@20": 0.01825656451103347, "mrr@10": 0.01814632936507937, "hitrate@10": 0.03175, "coverage@20": 0.0014754939697202976, "novelty@20": 11.940776484328246, "ild@20": 1, "tail_share@20": 0.0537875 }, "goodreads-poetry": { "ndcg@10": 0.12236973644812514, "recall@20": 0.1949647310300703, "mrr@10": 0.17403609664351852, "hitrate@10": 0.2877604166666667, "coverage@20": 0.011023079572855667, "novelty@20": 8.183416938392023, "ild@20": 0.5770357901589913, "tail_share@20": 0.04977213541666667 }, "hm-fashion": { "ndcg@10": 0.007965302247933582, "recall@20": 0.013275361652790942, "mrr@10": 0.01578095238095238, "hitrate@10": 0.04375, "coverage@20": 0.0007253808249330899, "novelty@20": 10.458076838222471, "ild@20": 0.7552628420343408, "tail_share@20": 0 }, "kuairand-pure": { "ndcg@10": 0.010508457592752492, "recall@20": 0.024142173233693047, "mrr@10": 0.020102083333333333, "hitrate@10": 0.05225, "coverage@20": 0.005305743467303356, "novelty@20": 8.20939485331943, "ild@20": 0.427259210526316, "tail_share@20": 0 }, "kuairec": { "ndcg@10": 0.07026721652772842, "recall@20": 0.08459300022026743, "mrr@10": 0.1270155933151528, "hitrate@10": 0.25726872246696036, "coverage@20": 0.04228445908841296, "novelty@20": 8.020988035780418, "ild@20": 0.781470325791656, "tail_share@20": 0 }, "lastfm-1k": { "ndcg@10": 0.06957820228628347, "recall@20": 0.013655280410670358, "mrr@10": 0.14603117008777386, "hitrate@10": 0.32220609579100146, "coverage@20": 0.006050940475632065, "novelty@20": 10.467415317134451, "ild@20": 1, "tail_share@20": 0 }, "mind-small": { "ndcg@10": 0.20642916323439603, "recall@20": 0.45051220737896436, "mrr@10": 0.22259761904761904, "hitrate@10": 0.48925, "coverage@20": 0.008353543519267044, "novelty@20": 8.425285075396163, "ild@20": 0.9227223684210526, "tail_share@20": 0 }, "ml-1m": { "ndcg@10": 0.18097750457449588, "recall@20": 0.09179191926236782, "mrr@10": 0.30926900222827436, "hitrate@10": 0.5849220103986135, "coverage@20": 0.04288, "novelty@20": 8.280971862936171, "ild@20": 0.7197513232906604, "tail_share@20": 0 }, "ml-32m": { "ndcg@10": 0.11468810692943299, "recall@20": 0.07577792836791483, "mrr@10": 0.21539722903359268, "hitrate@10": 0.45454545454545453, "coverage@20": 0.014413898434516991, "novelty@20": 9.413045864988861, "ild@20": 0.7268171953857924, "tail_share@20": 0 }, "retailrocket": { "ndcg@10": 0.011986034913725015, "recall@20": 0.02676005747126437, "mrr@10": 0.017241379310344827, "hitrate@10": 0.017241379310344827, "coverage@20": 0.01020408163265306, "novelty@20": 9.047297181990192, "ild@20": 1, "tail_share@20": 0 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 5, "compared_on_n_datasets": 10 }, { "rank": 8, "key": "lightgcn", "system": "LightGCN", "index": 59.84, "family": "graph", "generative": false, "confidence": "reproduction", "origin": "Simplified graph convolution for collaborative filtering", "citation": "He et al., 'LightGCN: Simplifying and Powering Graph Convolution Network for Recommendation', SIGIR 2020.", "summary": "Rounds of neighbour averaging over the user-item bipartite graph, with the nonlinearities and feature transforms of general graph convolution removed (He et al., SIGIR 2020). Propagation reaches items two or more hops away, which no direct co-occurrence connects to the user; on a sparse graph those hops tend to arrive at broadly popular items.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 65.78, "cold_start": 54.36, "long_tail": 60.4, "beyond_accuracy": 58.14, "multi_objective": 38.97, "robustness": 65.96 }, "per_dataset_relevance": { "amazon-beauty-2023": 37.17391304347826, "goodreads-poetry": 94.67391304347825, "hm-fashion": 80.5952380952381, "kuairand-pure": 78.15217391304348, "kuairec": 20.10869565217391, "lastfm-1k": 91.42857142857143, "mind-small": 38.58695652173913, "ml-1m": 80.65217391304348, "ml-32m": 54.02173913043478, "retailrocket": 82.3913043478261 }, "n_datasets": 10, "spread": 24.78766008337589, "monthly_cost_usd": 1.876574508258169, "train_cost_usd": 0.11258019086588875, "inference_cost_per_1k": 0.0001386850677991553, "latency_p50_ms": 0.0877892106901783, "latency_p95_ms": 0.0877892106901783, "model_mb": 5227.0462212, "requires_gpu": true, "retrain_note": "full retrain, or warm-start continuation from the last checkpoint", "latency_caveat": "", "value_index": 229.59472889414624, "other_track_index": 57.89, "win_rate": 60.12194556498701, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.007219419201085, "recall@20": 0.015968085818831947, "mrr@10": 0.007934027777777778, "hitrate@10": 0.023, "coverage@20": 0.5340005132152938, "novelty@20": 11.889133987830519, "ild@20": 1, "tail_share@20": 0.0426375 }, "goodreads-poetry": { "ndcg@10": 0.08294957604906816, "recall@20": 0.16719966836622943, "mrr@10": 0.10996248759920635, "hitrate@10": 0.234375, "coverage@20": 0.6930761281433, "novelty@20": 9.405891722023648, "ild@20": 0.4635545161732457, "tail_share@20": 0.06142578125 }, "hm-fashion": { "ndcg@10": 0.008780587999060112, "recall@20": 0.015143402986331229, "mrr@10": 0.018669742063492065, "hitrate@10": 0.04475, "coverage@20": 0.14157432652142374, "novelty@20": 11.963009875466028, "ild@20": 0.698529013713528, "tail_share@20": 0.0142875 }, "kuairand-pure": { "ndcg@10": 0.016447176531529106, "recall@20": 0.04204578286976417, "mrr@10": 0.02482013888888889, "hitrate@10": 0.0755, "coverage@20": 0.22337179997347129, "novelty@20": 9.554405873830412, "ild@20": 0.42358092105263173, "tail_share@20": 0.0047 }, "kuairec": { "ndcg@10": 0.03611708038429103, "recall@20": 0.051714549202091877, "mrr@10": 0.05924795468848332, "hitrate@10": 0.13392070484581498, "coverage@20": 0.6803953871499177, "novelty@20": 8.968964904415943, "ild@20": 0.8219275551682702, "tail_share@20": 0.039118942731277535 }, "lastfm-1k": { "ndcg@10": 0.08463504971832904, "recall@20": 0.009936668297392459, "mrr@10": 0.1795920012901145, "hitrate@10": 0.3671988388969521, "coverage@20": 0.07779445564988977, "novelty@20": 11.907425356449226, "ild@20": 1, "tail_share@20": 0.007402031930333817 }, "mind-small": { "ndcg@10": 0.09986625336312921, "recall@20": 0.2161832442776522, "mrr@10": 0.11528353174603174, "hitrate@10": 0.253, "coverage@20": 0.47722985718135275, "novelty@20": 9.502934123971166, "ild@20": 0.8874019736842106, "tail_share@20": 0.0135125 }, "ml-1m": { "ndcg@10": 0.17847317491188702, "recall@20": 0.10094448989647889, "mrr@10": 0.3080719925173998, "hitrate@10": 0.6065857885615251, "coverage@20": 0.37536, "novelty@20": 9.348999433170635, "ild@20": 0.6382310869755692, "tail_share@20": 0.0022097053726169843 }, "ml-32m": { "ndcg@10": 0.09739544191662376, "recall@20": 0.05640541876270273, "mrr@10": 0.18712575303484397, "hitrate@10": 0.38652256834075016, "coverage@20": 0.09507445589919816, "novelty@20": 9.372516277618859, "ild@20": 0.6720787060323705, "tail_share@20": 0 }, "retailrocket": { "ndcg@10": 0.01741281141911592, "recall@20": 0.05028735632183909, "mrr@10": 0.019636015325670497, "hitrate@10": 0.04310344827586207, "coverage@20": 0.5408163265306123, "novelty@20": 11.25042277947731, "ild@20": 1, "tail_share@20": 0.6090517241379311 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 2, "compared_on_n_datasets": 10 }, { "rank": 9, "key": "itemknn", "system": "ItemKNN", "index": 58.13, "family": "neighbourhood", "generative": false, "confidence": "reference", "origin": "Item-based collaborative filtering", "citation": "Sarwar et al., WWW 2001; shrinkage per Bell & Koren, ICDM 2007.", "summary": "Cosine item-item similarity with shrinkage, scoring the whole catalogue from the items already in a user's history. It generalises only through co-occurrence, so it makes no prediction about a pair of items that never appear together in the training data. It is deterministic, carries no random seed and runs on every dataset in the panel, which is why every release runs it as a fixed point of comparison; it sets no endpoint on the index scale and is scored like every other entrant.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 67.91, "cold_start": 50.32, "long_tail": 52.12, "beyond_accuracy": 41.56, "multi_objective": 46.79, "robustness": 67.69 }, "per_dataset_relevance": { "amazon-beauty-2023": 49.56521739130434, "goodreads-poetry": 52.065217391304344, "hm-fashion": 87.5, "kuairand-pure": 64.67391304347825, "kuairec": 41.08695652173913, "lastfm-1k": 91.96428571428571, "mind-small": 54.45652173913044, "ml-1m": 89.45652173913044, "ml-32m": 57.60869565217392, "retailrocket": 90.76086956521739 }, "n_datasets": 10, "spread": 18.850571630424312, "monthly_cost_usd": 4.1850188816937495, "train_cost_usd": 0.0013884152754010612, "inference_cost_per_1k": 0.0004178979275245756, "latency_p50_ms": 0.24974761046791322, "latency_p95_ms": 0.24974761046791322, "model_mb": 34.0507544, "requires_gpu": false, "retrain_note": "full item-item recompute, or incremental co-occurrence updates", "latency_caveat": "", "value_index": 100, "other_track_index": 57.84, "win_rate": 57.36426021236457, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.008631922985996486, "recall@20": 0.01806379207513603, "mrr@10": 0.011928869047619048, "hitrate@10": 0.02475, "coverage@20": 0.7153579676674365, "novelty@20": 12.307611405579816, "ild@20": 1, "tail_share@20": 0.1727 }, "goodreads-poetry": { "ndcg@10": 0.06609502835153398, "recall@20": 0.1289375356031245, "mrr@10": 0.0860684833829365, "hitrate@10": 0.18424479166666666, "coverage@20": 0.33000344471236653, "novelty@20": 7.906624523112332, "ild@20": 0.43230751439144743, "tail_share@20": 0.0119140625 }, "hm-fashion": { "ndcg@10": 0.009544831845529962, "recall@20": 0.01646241229395299, "mrr@10": 0.019242757936507936, "hitrate@10": 0.05225, "coverage@20": 0.178943945571425, "novelty@20": 11.743594443719275, "ild@20": 0.7068280734123851, "tail_share@20": 0.0138875 }, "kuairand-pure": { "ndcg@10": 0.015363445361626942, "recall@20": 0.03430890059065197, "mrr@10": 0.025965376984126983, "hitrate@10": 0.068, "coverage@20": 0.09324844143785647, "novelty@20": 8.680915973813649, "ild@20": 0.41750131578947386, "tail_share@20": 0.00025 }, "kuairec": { "ndcg@10": 0.04852452028052566, "recall@20": 0.05694442853325927, "mrr@10": 0.08837948395217118, "hitrate@10": 0.15594713656387665, "coverage@20": 0.047226798462383306, "novelty@20": 7.938864756546144, "ild@20": 0.7905528165374438, "tail_share@20": 0 }, "lastfm-1k": { "ndcg@10": 0.07441589819843293, "recall@20": 0.011846445978587581, "mrr@10": 0.15418596078973437, "hitrate@10": 0.33236574746008707, "coverage@20": 0.00973310192785778, "novelty@20": 10.46805257156884, "ild@20": 1, "tail_share@20": 0 }, "mind-small": { "ndcg@10": 0.11595377169132622, "recall@20": 0.26435900471969553, "mrr@10": 0.13141160714285713, "hitrate@10": 0.28925, "coverage@20": 0.430611694960927, "novelty@20": 8.757101228406466, "ild@20": 0.8887802631578947, "tail_share@20": 0.0112 }, "ml-1m": { "ndcg@10": 0.19800911330435214, "recall@20": 0.08913285575626495, "mrr@10": 0.3451813567714781, "hitrate@10": 0.6022530329289428, "coverage@20": 0.07584, "novelty@20": 8.442398618272996, "ild@20": 0.7248506905636768, "tail_share@20": 0 }, "ml-32m": { "ndcg@10": 0.1000088159868341, "recall@20": 0.05435418647517689, "mrr@10": 0.1923374605192787, "hitrate@10": 0.3782581055308328, "coverage@20": 0.038946162657502864, "novelty@20": 8.893062251776868, "ild@20": 0.7097049920262969, "tail_share@20": 0.00003178639542275906 }, "retailrocket": { "ndcg@10": 0.027601345076284232, "recall@20": 0.04249281609195402, "mrr@10": 0.03189655172413793, "hitrate@10": 0.05172413793103448, "coverage@20": 0.24489795918367346, "novelty@20": 10.836354743185085, "ild@20": 1, "tail_share@20": 0.4495689655172414 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 4, "compared_on_n_datasets": 10 }, { "rank": 10, "key": "bert4rec", "system": "BERT4Rec", "index": 57.19, "family": "sequential", "generative": false, "confidence": "reproduction", "origin": "Bidirectional cloze-objective sequential recommendation", "citation": "Sun et al., 'BERT4Rec', CIKM 2019; training regime per Petrov & Macdonald, RecSys 2022.", "summary": "A bidirectional transformer trained to fill in masked positions, queried at serving time with a mask token appended to the history (Sun et al., CIKM 2019). Petrov & Macdonald (RecSys 2022) could not reproduce its published numbers at its published settings and recovered them only with far longer training, so it is reported under both compute tracks.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 58.09, "cold_start": 57.68, "long_tail": 50.15, "beyond_accuracy": 51.98, "multi_objective": 67.75, "robustness": 58.06 }, "per_dataset_relevance": { "amazon-beauty-2023": 85.65217391304348, "goodreads-poetry": 64.89130434782608, "hm-fashion": 47.142857142857146, "kuairand-pure": 46.73913043478261, "kuairec": 77.3913043478261, "lastfm-1k": 20.952380952380953, "mind-small": 79.23913043478261, "ml-1m": 30.434782608695652, "ml-32m": 92.5, "retailrocket": 35.97826086956522 }, "n_datasets": 10, "spread": 23.856718708413656, "monthly_cost_usd": 1.7143458886454561, "train_cost_usd": 0.069747054493166, "inference_cost_per_1k": 0.00014109462016001836, "latency_p50_ms": 0.08733716307731042, "latency_p95_ms": 0.1138824562986177, "model_mb": 8.26662, "requires_gpu": true, "retrain_note": "full retrain, or warm-start continuation from the last checkpoint", "latency_caveat": "", "value_index": 240.18629832127954, "other_track_index": 37.11, "win_rate": 55.26266981963474, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.011625749482506763, "recall@20": 0.022817019024191188, "mrr@10": 0.017609424603174603, "hitrate@10": 0.03, "coverage@20": 0.0032075955863484734, "novelty@20": 9.746572782482898, "ild@20": 1, "tail_share@20": 0.0001 }, "goodreads-poetry": { "ndcg@10": 0.06783914059767854, "recall@20": 0.15245479196025472, "mrr@10": 0.08862847222222221, "hitrate@10": 0.21614583333333334, "coverage@20": 0.601791250430589, "novelty@20": 10.353668609348079, "ild@20": 0.4892376816063597, "tail_share@20": 0.1455078125 }, "hm-fashion": { "ndcg@10": 0.006087679830425097, "recall@20": 0.010736421867464997, "mrr@10": 0.011736408730158729, "hitrate@10": 0.03525, "coverage@20": 0.001875984892068336, "novelty@20": 11.030926721966432, "ild@20": 0.800848169150908, "tail_share@20": 0 }, "kuairand-pure": { "ndcg@10": 0.012425180997003491, "recall@20": 0.03338747580990641, "mrr@10": 0.019341666666666667, "hitrate@10": 0.06125, "coverage@20": 0.02109033028253084, "novelty@20": 9.20136353026922, "ild@20": 0.41167434210526327, "tail_share@20": 0 }, "kuairec": { "ndcg@10": 0.058135332522330926, "recall@20": 0.07645409185969639, "mrr@10": 0.12229913992028528, "hitrate@10": 0.28193832599118945, "coverage@20": 0.1543108182317408, "novelty@20": 9.931439148730753, "ild@20": 0.5248443948452736, "tail_share@20": 0.11854625550660793 }, "lastfm-1k": { "ndcg@10": 0.012986182345844645, "recall@20": 0.001653438998896042, "mrr@10": 0.033325846061695114, "hitrate@10": 0.11030478955007257, "coverage@20": 0.04692996857263474, "novelty@20": 13.50809343297765, "ild@20": 1, "tail_share@20": 0.08911465892597968 }, "mind-small": { "ndcg@10": 0.1361313529553007, "recall@20": 0.3072411924515543, "mrr@10": 0.15026418650793652, "hitrate@10": 0.3375, "coverage@20": 0.13123147399622742, "novelty@20": 8.632864412365212, "ild@20": 0.9050203947368421, "tail_share@20": 0.00095 }, "ml-1m": { "ndcg@10": 0.12382223588911741, "recall@20": 0.07355933702826255, "mrr@10": 0.24131110560919924, "hitrate@10": 0.48353552859618715, "coverage@20": 0.5152, "novelty@20": 10.402192709860625, "ild@20": 0.5798352426762234, "tail_share@20": 0.046967071057192375 }, "ml-32m": { "ndcg@10": 0.11378392919622637, "recall@20": 0.07537339305859123, "mrr@10": 0.2303012643921735, "hitrate@10": 0.47425301970756517, "coverage@20": 0.11989308896525391, "novelty@20": 11.626924786898241, "ild@20": 0.6432075388074204, "tail_share@20": 0.00371900826446281 }, "retailrocket": { "ndcg@10": 0.009641935923681257, "recall@20": 0.02011494252873563, "mrr@10": 0.010057471264367814, "hitrate@10": 0.034482758620689655, "coverage@20": 0.5176252319109462, "novelty@20": 11.155179949117757, "ild@20": 1, "tail_share@20": 0.5685344827586207 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 3, "compared_on_n_datasets": 10 }, { "rank": 11, "key": "plum_style", "system": "PLUM-style SID generative retrieval", "index": 55.05, "family": "generative-semantic-id", "generative": true, "confidence": "proxy", "origin": "Google / YouTube — PLUM", "citation": "Sohn et al., 'PLUM: Adapting Pre-trained Language Models for Industrial-scale Generative Recommendations', arXiv:2510.07784 / WWW 2026.", "summary": "Semantic-ID tokenization, continued pre-training on the domain's own token streams, then fine-tuning for generative retrieval. The decoder-only model is trained from scratch at benchmark scale rather than adapted from a pre-trained language model, so the transfer the published method relies on is absent here; what is measured is the architecture and the training curriculum.", "stages": "continued pre-training + generative retrieval + rescoring", "proxy_of": "PLUM (Google/YouTube)", "proxy_caveat": "PLUM adapts a large pre-trained language model and a multimodal item encoder on YouTube-scale logs. This entrant trains a small decoder-only transformer from scratch on public data with a TF-IDF/SVD content encoder. It shares the mechanism, not the scale, the data, or the pre-training.", "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 50.56, "cold_start": 52.15, "long_tail": 65.46, "beyond_accuracy": 58.84, "multi_objective": 57.34, "robustness": 53.96 }, "per_dataset_relevance": { "amazon-beauty-2023": 31.73913043478261, "goodreads-poetry": 46.52173913043478, "hm-fashion": 45.952380952380956, "kuairand-pure": 82.17391304347827, "kuairec": 36.086956521739125, "lastfm-1k": 44.88095238095238, "mind-small": 75.21739130434784, "ml-1m": 26.413043478260867, "ml-32m": 66.6304347826087, "retailrocket": 50.00000000000001 }, "n_datasets": 10, "spread": 17.557214490114752, "monthly_cost_usd": 91.7980982928182, "train_cost_usd": 0.16427903252668535, "inference_cost_per_1k": 0.009108348450132711, "latency_p50_ms": 4.814468809522519, "latency_p95_ms": 4.814468809522519, "model_mb": 5.0630375999999995, "requires_gpu": true, "retrain_note": "tokenizer refit when the catalogue shifts, then a model retrain; the tokenizer is the harder half because changing item codes invalidates the model", "latency_caveat": "", "value_index": 4.317733483398226, "other_track_index": 46.03, "win_rate": 54.52659155191113, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.006627570245643457, "recall@20": 0.014982637856865108, "mrr@10": 0.008079662698412697, "hitrate@10": 0.02125, "coverage@20": 0.18148575827559663, "novelty@20": 11.526209299447286, "ild@20": 1, "tail_share@20": 0.026725 }, "goodreads-poetry": { "ndcg@10": 0.06006612101389561, "recall@20": 0.13116536960560826, "mrr@10": 0.0811846374834656, "hitrate@10": 0.19010416666666666, "coverage@20": 0.6996210816396831, "novelty@20": 10.090471265140208, "ild@20": 0.46995871025219316, "tail_share@20": 0.12708333333333333 }, "hm-fashion": { "ndcg@10": 0.005719845682127772, "recall@20": 0.010996590891297224, "mrr@10": 0.011309821428571428, "hitrate@10": 0.02975, "coverage@20": 0.28640036018909926, "novelty@20": 13.535694997583063, "ild@20": 0.5203328036864595, "tail_share@20": 0.0866 }, "kuairand-pure": { "ndcg@10": 0.017371048243750575, "recall@20": 0.03800749844998254, "mrr@10": 0.027616765873015873, "hitrate@10": 0.071, "coverage@20": 0.14484679665738162, "novelty@20": 10.168680909532146, "ild@20": 0.42230526315789485, "tail_share@20": 0.0061375 }, "kuairec": { "ndcg@10": 0.041461863147960104, "recall@20": 0.05867514005129643, "mrr@10": 0.09239214040976156, "hitrate@10": 0.20792951541850221, "coverage@20": 0.40911587040087866, "novelty@20": 9.607116149173754, "ild@20": 0.6002186905428115, "tail_share@20": 0.054889867841409694 }, "lastfm-1k": { "ndcg@10": 0.030345357454124377, "recall@20": 0.007460290493504175, "mrr@10": 0.07028877830764624, "hitrate@10": 0.19593613933236576, "coverage@20": 0.07920165110933909, "novelty@20": 12.191726730617292, "ild@20": 1, "tail_share@20": 0.05377358490566038 }, "mind-small": { "ndcg@10": 0.13246719352972047, "recall@20": 0.2907201516867635, "mrr@10": 0.15453551587301587, "hitrate@10": 0.3295, "coverage@20": 0.2950687146321746, "novelty@20": 9.045501799573813, "ild@20": 0.9007611842105263, "tail_share@20": 0.00785 }, "ml-1m": { "ndcg@10": 0.11919018755690701, "recall@20": 0.07023503465297014, "mrr@10": 0.23230241258837447, "hitrate@10": 0.48440207972270366, "coverage@20": 0.41376, "novelty@20": 10.039691461819373, "ild@20": 0.58436938318753, "tail_share@20": 0.02647313691507799 }, "ml-32m": { "ndcg@10": 0.094872409834551, "recall@20": 0.06264298978387164, "mrr@10": 0.20671348853167038, "hitrate@10": 0.43738080101716464, "coverage@20": 0.10080183276059565, "novelty@20": 11.413293216737411, "ild@20": 0.6642728185612362, "tail_share@20": 0.014335664335664336 }, "retailrocket": { "ndcg@10": 0.011130597859511674, "recall@20": 0.034482758620689655, "mrr@10": 0.009195402298850575, "hitrate@10": 0.034482758620689655, "coverage@20": 0.5468460111317254, "novelty@20": 10.909327071595264, "ild@20": 1, "tail_share@20": 0.4625 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 4, "compared_on_n_datasets": 10 }, { "rank": 12, "key": "twotower_dcnv2", "system": "Two-Tower + DCN-v2", "index": 53.56, "family": "multi-stage-neural", "generative": false, "confidence": "reference", "origin": "The standard industrial retrieval-plus-ranking stack", "citation": "Yi et al., 'Sampling-Bias-Corrected Neural Modeling for Large Corpus Item Recommendations', RecSys 2019 (retrieval); Wang et al., 'DCN V2', WWW 2021 (ranking).", "summary": "A two-tower encoder trained with a logQ-corrected sampled softmax produces a few hundred candidates, and a deep-and- cross network reorders them (Yi et al., RecSys 2019; Wang et al., WWW 2021). This is the shape most large-scale production recommenders are built in.", "stages": "retrieval + ranking", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": "twotower", "retriever_label": null, "ranker_label": "DCN-v2 ranker", "is_new": false, "dimensions": { "relevance": 52.41, "cold_start": 58.08, "long_tail": 50.11, "beyond_accuracy": 44.56, "multi_objective": 61.63, "robustness": 55.8 }, "per_dataset_relevance": { "amazon-beauty-2023": 90.32608695652175, "goodreads-poetry": 68.04347826086956, "hm-fashion": 73.69047619047619, "kuairand-pure": 40.108695652173914, "kuairec": 8.804347826086955, "lastfm-1k": 48.33333333333333, "mind-small": 57.28260869565217, "ml-1m": 54.565217391304344, "ml-32m": 52.826086956521735, "retailrocket": 30.10869565217391 }, "n_datasets": 10, "spread": 21.749015449522975, "monthly_cost_usd": 57.91770840198719, "train_cost_usd": 0.03455094847049578, "inference_cost_per_1k": 0.005776741177614055, "latency_p50_ms": 3.0675376652749504, "latency_p95_ms": 3.0675376652749504, "model_mb": 22.9052708, "requires_gpu": true, "retrain_note": "retriever refit on train+valid for serving; ranker weights carried over from the valid-window fit", "latency_caveat": "includes constructing the ranker's features online, in Python, for every request — work a deployed pipeline precomputes or caches, so this is an upper bound", "value_index": 6.658327408725262, "other_track_index": 56.69, "win_rate": 49.18509760282184, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.012392088447482793, "recall@20": 0.021886287057329798, "mrr@10": 0.01816795634920635, "hitrate@10": 0.03375, "coverage@20": 0.013279445727482679, "novelty@20": 9.955848251307856, "ild@20": 1, "tail_share@20": 0.0000375 }, "goodreads-poetry": { "ndcg@10": 0.06804370142959602, "recall@20": 0.15619088118731916, "mrr@10": 0.08929604828042327, "hitrate@10": 0.19921875, "coverage@20": 0.2542197726489838, "novelty@20": 8.599737734218449, "ild@20": 0.4384773077684992, "tail_share@20": 0.056868489583333334 }, "hm-fashion": { "ndcg@10": 0.00806584568245662, "recall@20": 0.014416328665673944, "mrr@10": 0.016989484126984127, "hitrate@10": 0.04575, "coverage@20": 0.15958378148527977, "novelty@20": 12.319220138209378, "ild@20": 0.5750781042352678, "tail_share@20": 0.0899125 }, "kuairand-pure": { "ndcg@10": 0.011444757292187186, "recall@20": 0.031010203368199023, "mrr@10": 0.017533630952380953, "hitrate@10": 0.052, "coverage@20": 0.08582040058363179, "novelty@20": 9.26514766416221, "ild@20": 0.372759210526316, "tail_share@20": 0 }, "kuairec": { "ndcg@10": 0.015442283065724016, "recall@20": 0.03142094415565832, "mrr@10": 0.03148486119851759, "hitrate@10": 0.1092511013215859, "coverage@20": 0.07303679297089512, "novelty@20": 10.952521714991086, "ild@20": 0.6979531962492909, "tail_share@20": 0.22290748898678414 }, "lastfm-1k": { "ndcg@10": 0.03697428806428533, "recall@20": 0.005840245737921111, "mrr@10": 0.08628159052687355, "hitrate@10": 0.22641509433962265, "coverage@20": 0.054857169660865894, "novelty@20": 11.535918034311162, "ild@20": 1, "tail_share@20": 0.03541364296081277 }, "mind-small": { "ndcg@10": 0.11373839590501995, "recall@20": 0.29290579725035026, "mrr@10": 0.12135753968253968, "hitrate@10": 0.31625, "coverage@20": 0.039881433575855565, "novelty@20": 8.988550197186669, "ild@20": 0.8634947368421053, "tail_share@20": 0.0000375 }, "ml-1m": { "ndcg@10": 0.16541596726998367, "recall@20": 0.07921818528210797, "mrr@10": 0.30214196308217106, "hitrate@10": 0.5623916811091855, "coverage@20": 0.1904, "novelty@20": 8.844699469957456, "ild@20": 0.6547218810118146, "tail_share@20": 0.0012998266897746968 }, "ml-32m": { "ndcg@10": 0.08150093108717563, "recall@20": 0.061476706005688886, "mrr@10": 0.16301980847435393, "hitrate@10": 0.3954227590591227, "coverage@20": 0.0892516227567774, "novelty@20": 11.174638044830035, "ild@20": 0.5929583582826023, "tail_share@20": 0.002002542911633821 }, "retailrocket": { "ndcg@10": 0.01074398610983388, "recall@20": 0.01293103448275862, "mrr@10": 0.01293103448275862, "hitrate@10": 0.017241379310344827, "coverage@20": 0.3293135435992579, "novelty@20": 10.840258830859128, "ild@20": 1, "tail_share@20": 0.4413793103448276 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 5, "compared_on_n_datasets": 10 }, { "rank": 13, "key": "gru4rec", "system": "GRU4Rec", "index": 52.19, "family": "sequential", "generative": false, "confidence": "reproduction", "origin": "Recurrent session-based recommendation", "citation": "Hidasi et al., 'Session-based Recommendations with Recurrent Neural Networks', ICLR 2016.", "summary": "A gated recurrent network over the ordered item sequence (Hidasi et al., ICLR 2016), the first neural sequential recommender. It is trained here with the same objective, windowing and compute budget as every other sequential entrant.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 38.94, "cold_start": 48.14, "long_tail": 78.31, "beyond_accuracy": 72.62, "multi_objective": 57.34, "robustness": 43.97 }, "per_dataset_relevance": { "amazon-beauty-2023": 17.065217391304344, "goodreads-poetry": 33.26086956521739, "hm-fashion": 23.452380952380953, "kuairand-pure": 76.52173913043478, "kuairec": 52.391304347826086, "lastfm-1k": 24.166666666666664, "mind-small": 48.58695652173913, "ml-1m": 11.630434782608695, "ml-32m": 86.08695652173913, "retailrocket": 16.195652173913043 }, "n_datasets": 10, "spread": 24.781480742331276, "monthly_cost_usd": 1.419418381021487, "train_cost_usd": 0.03632750316247545, "inference_cost_per_1k": 0.00012613937422647185, "latency_p50_ms": 0.07958613044065854, "latency_p95_ms": 0.11118025458993233, "model_mb": 7.049596000000001, "requires_gpu": true, "retrain_note": "full retrain, or warm-start continuation from the last checkpoint", "latency_caveat": "", "value_index": 264.7163761566911, "other_track_index": 49.54, "win_rate": 48.54639031854493, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.0028046051157846834, "recall@20": 0.007447347871191829, "mrr@10": 0.003229861111111111, "hitrate@10": 0.012, "coverage@20": 0.611367718758019, "novelty@20": 12.086300150024087, "ild@20": 1, "tail_share@20": 0.0788875 }, "goodreads-poetry": { "ndcg@10": 0.054967496844214786, "recall@20": 0.11962288789788189, "mrr@10": 0.07499431630291005, "hitrate@10": 0.169921875, "coverage@20": 0.8983809851877368, "novelty@20": 10.231059303198291, "ild@20": 0.5003889117324563, "tail_share@20": 0.15511067708333334 }, "hm-fashion": { "ndcg@10": 0.0021237139984091255, "recall@20": 0.004686866729941168, "mrr@10": 0.004424007936507938, "hitrate@10": 0.01425, "coverage@20": 0.40848945696490657, "novelty@20": 14.570414553980253, "ild@20": 0.777237738138904, "tail_share@20": 0.1685 }, "kuairand-pure": { "ndcg@10": 0.01568193598255609, "recall@20": 0.04239615888409075, "mrr@10": 0.02393928571428571, "hitrate@10": 0.074, "coverage@20": 0.5078922934076138, "novelty@20": 10.631591111751957, "ild@20": 0.4549355263157896, "tail_share@20": 0.0312 }, "kuairec": { "ndcg@10": 0.048370056753421105, "recall@20": 0.06203694656224813, "mrr@10": 0.10500454513670372, "hitrate@10": 0.239647577092511, "coverage@20": 0.43657331136738053, "novelty@20": 9.797989649073715, "ild@20": 0.6807123840611493, "tail_share@20": 0.10731277533039647 }, "lastfm-1k": { "ndcg@10": 0.016432880318173377, "recall@20": 0.001203267064202353, "mrr@10": 0.042199990784896446, "hitrate@10": 0.11175616835994194, "coverage@20": 0.13438716637740983, "novelty@20": 13.771952161375424, "ild@20": 1, "tail_share@20": 0.12285921625544267 }, "mind-small": { "ndcg@10": 0.11203582438819258, "recall@20": 0.25311716187686584, "mrr@10": 0.12918194444444445, "hitrate@10": 0.2895, "coverage@20": 0.506332524925896, "novelty@20": 9.356529909124804, "ild@20": 0.9077973684210526, "tail_share@20": 0.0177 }, "ml-1m": { "ndcg@10": 0.10544635768927904, "recall@20": 0.061735463881453456, "mrr@10": 0.2077831421418943, "hitrate@10": 0.4610051993067591, "coverage@20": 0.7264, "novelty@20": 10.428799545507557, "ild@20": 0.6247592965831295, "tail_share@20": 0.07079722703639514 }, "ml-32m": { "ndcg@10": 0.10724442100904234, "recall@20": 0.06877918730977986, "mrr@10": 0.22116948707857798, "hitrate@10": 0.45200254291163383, "coverage@20": 0.2686139747995418, "novelty@20": 12.054828452005522, "ild@20": 0.651149049270706, "tail_share@20": 0.04459631277813096 }, "retailrocket": { "ndcg@10": 0.0052857516617711925, "recall@20": 0.004310344827586207, "mrr@10": 0.008620689655172414, "hitrate@10": 0.008620689655172414, "coverage@20": 0.6168831168831169, "novelty@20": 11.089472539141717, "ild@20": 1, "tail_share@20": 0.5456896551724137 } }, "beats_next_significantly": null, "beats_next_on_n_datasets": null, "compared_on_n_datasets": null }, { "rank": 14, "key": "option_cascade", "system": "Cheap full pass + Option-Attention reranker", "index": 52.02, "family": "multi-stage-neural", "generative": false, "confidence": "reproduction", "origin": "Full-catalogue cheap scoring feeding a parallel option reranker", "citation": "Zhou et al., KDD 2018 (target attention); Yi et al., RecSys 2019 (the pooled two-tower stage this degenerates to).", "summary": "A cheap dot-product pass scores the whole catalogue, the top few hundred are kept, and an option-attention head rescores those. Both stages are the same model — the cheap stage is the option head with the per-item attention removed — so they share one embedding table and one training run.", "stages": "cheap full-catalogue scoring + option-attention ranking", "proxy_of": null, "proxy_caveat": null, "introduced_in": "v3.2", "native_retriever": null, "retriever_label": "Option-Attention cheap pass", "ranker_label": "Option-Attention reranker (cascade-trained)", "is_new": true, "dimensions": { "relevance": 51.25, "cold_start": 48.91, "long_tail": 50.16, "beyond_accuracy": 46.12, "multi_objective": 65.16, "robustness": 53.96 }, "per_dataset_relevance": { "amazon-beauty-2023": 40.97826086956521, "goodreads-poetry": 50.54347826086956, "hm-fashion": 43.452380952380956, "kuairand-pure": 94.45652173913042, "kuairec": 47.82608695652174, "lastfm-1k": 55.476190476190474, "mind-small": 64.34782608695652, "ml-1m": 30, "ml-32m": 48.80434782608695, "retailrocket": 36.630434782608695 }, "n_datasets": 10, "spread": 17.047071518596418, "monthly_cost_usd": 3.8966801284703765, "train_cost_usd": 0.1797072397426434, "inference_cost_per_1k": 0.0003114953635589877, "latency_p50_ms": 0.15882569414514863, "latency_p95_ms": 0.1640036502351754, "model_mb": 16.948072, "requires_gpu": true, "retrain_note": "two fits per cycle: the retriever nightly, the ranker on the newer window; both are charged here", "latency_caveat": "includes constructing the ranker's features online, in Python, for every request — work a deployed pipeline precomputes or caches, so this is an upper bound", "value_index": 96.12082882547067, "other_track_index": 33.76, "win_rate": 49.661519914687126, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.006586112144121495, "recall@20": 0.018878828543621284, "mrr@10": 0.00862986111111111, "hitrate@10": 0.02275, "coverage@20": 0.057993328201180395, "novelty@20": 10.077977219187044, "ild@20": 1, "tail_share@20": 0.00325 }, "goodreads-poetry": { "ndcg@10": 0.06301065378539816, "recall@20": 0.13335692997127727, "mrr@10": 0.0823585792824074, "hitrate@10": 0.19401041666666666, "coverage@20": 0.4102652428522218, "novelty@20": 9.037589137530286, "ild@20": 0.466420384457237, "tail_share@20": 0.03626302083333333 }, "hm-fashion": { "ndcg@10": 0.005015206696320805, "recall@20": 0.00961357527462168, "mrr@10": 0.009770734126984128, "hitrate@10": 0.02975, "coverage@20": 0.13672177893394033, "novelty@20": 12.384699087747025, "ild@20": 0.5438701896837227, "tail_share@20": 0.012675 }, "kuairand-pure": { "ndcg@10": 0.018171526532123665, "recall@20": 0.042155629652960935, "mrr@10": 0.028578174603174602, "hitrate@10": 0.081, "coverage@20": 0.07733121103594641, "novelty@20": 9.224411766726046, "ild@20": 0.4159111842105265, "tail_share@20": 0.0003625 }, "kuairec": { "ndcg@10": 0.04554186107547992, "recall@20": 0.06461026441611034, "mrr@10": 0.09870253828403608, "hitrate@10": 0.2343612334801762, "coverage@20": 0.2745744096650192, "novelty@20": 9.43923027727376, "ild@20": 0.6342957339763394, "tail_share@20": 0.051145374449339205 }, "lastfm-1k": { "ndcg@10": 0.045469937188655586, "recall@20": 0.007286817831947588, "mrr@10": 0.10828265026378234, "hitrate@10": 0.2656023222060958, "coverage@20": 0.055842206482480415, "novelty@20": 11.704060904712687, "ild@20": 1, "tail_share@20": 0.0059506531204644416 }, "mind-small": { "ndcg@10": 0.12363828298352453, "recall@20": 0.27635758357139933, "mrr@10": 0.1401470238095238, "hitrate@10": 0.31825, "coverage@20": 0.246833737537052, "novelty@20": 8.735862921712224, "ild@20": 0.9036427631578947, "tail_share@20": 0.002525 }, "ml-1m": { "ndcg@10": 0.13055758012588548, "recall@20": 0.06863142252449268, "mrr@10": 0.24616035322274493, "hitrate@10": 0.5086655112651647, "coverage@20": 0.38592, "novelty@20": 9.628858211441008, "ild@20": 0.589621907915997, "tail_share@20": 0.005805892547660312 }, "ml-32m": { "ndcg@10": 0.08801626201811587, "recall@20": 0.05764610017917181, "mrr@10": 0.17326688463052098, "hitrate@10": 0.38652256834075016, "coverage@20": 0.0892516227567774, "novelty@20": 10.06973074685063, "ild@20": 0.6370658950278008, "tail_share@20": 0.005562619198982835 }, "retailrocket": { "ndcg@10": 0.010757502793475805, "recall@20": 0.022988505747126436, "mrr@10": 0.007614942528735632, "hitrate@10": 0.034482758620689655, "coverage@20": 0.44990723562152135, "novelty@20": 10.59461891429591, "ild@20": 1, "tail_share@20": 0.34094827586206894 } }, "beats_next_significantly": null, "beats_next_on_n_datasets": null, "compared_on_n_datasets": null }, { "rank": 15, "key": "tiger", "system": "TIGER", "index": 51.6, "family": "generative-semantic-id", "generative": true, "confidence": "reproduction", "origin": "Google — Transformer Index for GEnerative Recommenders", "citation": "Rajput et al., 'Recommender Systems with Generative Retrieval', NeurIPS 2023 (arXiv:2305.05065).", "summary": "Items are quantised into short tuples of semantic codes by an RQ-VAE, and a sequence-to-sequence transformer decodes the codes of the next item autoregressively (Rajput et al., NeurIPS 2023). Items sharing a code prefix share a neighbourhood, so an item with no interaction history is reachable through its content alone. Decoding cost does not depend on catalogue size, and recall is bounded by the beam width rather than by a candidate budget.", "stages": "single-stage (generative retrieval)", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 48.25, "cold_start": 50.02, "long_tail": 59.13, "beyond_accuracy": 56.66, "multi_objective": 52.16, "robustness": 49.74 }, "per_dataset_relevance": { "amazon-beauty-2023": 66.73913043478261, "goodreads-poetry": 34.78260869565217, "hm-fashion": 40.595238095238095, "kuairand-pure": 71.6304347826087, "kuairec": 37.71739130434783, "lastfm-1k": 56.07142857142857, "mind-small": 62.93478260869565, "ml-1m": 18.913043478260867, "ml-32m": 52.71739130434782, "retailrocket": 40.43478260869565 }, "n_datasets": 10, "spread": 15.677704579888461, "monthly_cost_usd": 32.50427361931068, "train_cost_usd": 0.24390117211920082, "inference_cost_per_1k": 0.003144330352059215, "latency_p50_ms": 1.6707963065556828, "latency_p95_ms": 1.6707963065556828, "model_mb": 9.4185576, "requires_gpu": true, "retrain_note": "tokenizer refit when the catalogue shifts, then a model retrain; the tokenizer is the harder half because changing item codes invalidates the model", "latency_caveat": "", "value_index": 11.430747853954685, "other_track_index": 47.66, "win_rate": 48.91384986321974, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.009996357604614182, "recall@20": 0.021161334494131104, "mrr@10": 0.012994940476190476, "hitrate@10": 0.0295, "coverage@20": 0.11265075699255837, "novelty@20": 11.217310343942113, "ild@20": 1, "tail_share@20": 0.0221625 }, "goodreads-poetry": { "ndcg@10": 0.055181491534953164, "recall@20": 0.11843052670464917, "mrr@10": 0.0779754154265873, "hitrate@10": 0.17317708333333334, "coverage@20": 0.6575955907681709, "novelty@20": 9.95713195308908, "ild@20": 0.4621333607456141, "tail_share@20": 0.11194661458333334 }, "hm-fashion": { "ndcg@10": 0.004517793423918171, "recall@20": 0.00898287767855938, "mrr@10": 0.009002480158730158, "hitrate@10": 0.02325, "coverage@20": 0.24355286525425848, "novelty@20": 13.48965324530211, "ild@20": 0.4941348997868263, "tail_share@20": 0.0690625 }, "kuairand-pure": { "ndcg@10": 0.016578338730092058, "recall@20": 0.0373246631931549, "mrr@10": 0.02480238095238095, "hitrate@10": 0.06575, "coverage@20": 0.11168589998673564, "novelty@20": 10.062853074873324, "ild@20": 0.4156750000000001, "tail_share@20": 0.0022375 }, "kuairec": { "ndcg@10": 0.04350969864523755, "recall@20": 0.053019041416758755, "mrr@10": 0.098215859030837, "hitrate@10": 0.22466960352422907, "coverage@20": 0.4299835255354201, "novelty@20": 9.570081460088113, "ild@20": 0.6225989280466382, "tail_share@20": 0.04590308370044053 }, "lastfm-1k": { "ndcg@10": 0.04064659006463302, "recall@20": 0.008870738260974734, "mrr@10": 0.09280473195567535, "hitrate@10": 0.23947750362844702, "coverage@20": 0.056944509592382384, "novelty@20": 11.693057730686519, "ild@20": 1, "tail_share@20": 0.027213352685050797 }, "mind-small": { "ndcg@10": 0.12239382800317666, "recall@20": 0.2847291259179417, "mrr@10": 0.1385016865079365, "hitrate@10": 0.31525, "coverage@20": 0.26623551603341417, "novelty@20": 9.046000706114926, "ild@20": 0.9007394736842106, "tail_share@20": 0.0050875 }, "ml-1m": { "ndcg@10": 0.11651975859799317, "recall@20": 0.06676167360672836, "mrr@10": 0.21998328794256006, "hitrate@10": 0.47053726169844023, "coverage@20": 0.37888, "novelty@20": 10.045825711745097, "ild@20": 0.5838771046996304, "tail_share@20": 0.022400346620450606 }, "ml-32m": { "ndcg@10": 0.09029930154406028, "recall@20": 0.05685034119201663, "mrr@10": 0.19053799735617918, "hitrate@10": 0.43102352193261284, "coverage@20": 0.08419243986254296, "novelty@20": 11.141281184447134, "ild@20": 0.6706227236424311, "tail_share@20": 0.00909090909090909 }, "retailrocket": { "ndcg@10": 0.008408012890704376, "recall@20": 0.034482758620689655, "mrr@10": 0.0066091954022988505, "hitrate@10": 0.02586206896551724, "coverage@20": 0.5097402597402597, "novelty@20": 10.917278985915265, "ild@20": 1, "tail_share@20": 0.4650862068965517 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 3, "compared_on_n_datasets": 10 }, { "rank": 16, "key": "twotower_rankmixer", "system": "Two-Tower + RankMixer-style ranker", "index": 51.3, "family": "multi-stage-neural", "generative": false, "confidence": "proxy", "origin": "ByteDance — RankMixer", "citation": "Zhang et al., 'RankMixer: Scaling Up Ranking Models in Industrial Recommenders', CIKM 2025 (arXiv:2507.15551).", "summary": "A two-tower encoder produces the candidates, and the ranking stage replaces quadratic attention with a per- token feed-forward network and a parameter-free token- mixing step (ByteDance, CIKM 2025). The design targets hardware utilisation at large ranker sizes; here it is run small, which measures whether the substitution costs anything in that regime.", "stages": "retrieval + ranking", "proxy_of": "RankMixer (ByteDance)", "proxy_caveat": "RankMixer is a billion-parameter ranker over ByteDance's own sparse feature space with a sparse mixture-of-experts variant, trained on trillion-scale logs. This entrant implements the per-token feed-forward and token-mixing design over this benchmark's shared feature space, at a size where its scaling argument cannot be exercised.", "introduced_in": null, "native_retriever": "twotower", "retriever_label": null, "ranker_label": "RankMixer-style ranker", "is_new": false, "dimensions": { "relevance": 50.37, "cold_start": 56.68, "long_tail": 48.57, "beyond_accuracy": 42.64, "multi_objective": 56.45, "robustness": 53.14 }, "per_dataset_relevance": { "amazon-beauty-2023": 77.5, "goodreads-poetry": 65.54347826086956, "hm-fashion": 64.16666666666667, "kuairand-pure": 26.304347826086957, "kuairec": 31.630434782608692, "lastfm-1k": 43.92857142857143, "mind-small": 40.97826086956521, "ml-1m": 55.326086956521735, "ml-32m": 52.608695652173914, "retailrocket": 45.76086956521739 }, "n_datasets": 10, "spread": 15.05809235391532, "monthly_cost_usd": 59.380785703386515, "train_cost_usd": 0.03860493519966095, "inference_cost_per_1k": 0.005921285423526799, "latency_p50_ms": 3.144379166408765, "latency_p95_ms": 3.144379166408765, "model_mb": 24.9114188, "requires_gpu": true, "retrain_note": "retriever refit on train+valid for serving; ranker weights carried over from the valid-window fit", "latency_caveat": "includes constructing the ranker's features online, in Python, for every request — work a deployed pipeline precomputes or caches, so this is an upper bound", "value_index": 6.22007614538882, "other_track_index": 53.71, "win_rate": 47.12755598831815, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.01151165901283288, "recall@20": 0.020640661155453894, "mrr@10": 0.016244742063492062, "hitrate@10": 0.03325, "coverage@20": 0.01546061072619964, "novelty@20": 9.911279479931677, "ild@20": 1, "tail_share@20": 0.0011375 }, "goodreads-poetry": { "ndcg@10": 0.06823062024236214, "recall@20": 0.15505381048438185, "mrr@10": 0.08637152777777778, "hitrate@10": 0.19856770833333334, "coverage@20": 0.2349293833964864, "novelty@20": 8.489336515701607, "ild@20": 0.44296701195428034, "tail_share@20": 0.04635416666666667 }, "hm-fashion": { "ndcg@10": 0.0074171507171370285, "recall@20": 0.014467182695729136, "mrr@10": 0.015179067460317459, "hitrate@10": 0.04, "coverage@20": 0.1355211486030166, "novelty@20": 12.30761592649567, "ild@20": 0.5571609742587368, "tail_share@20": 0.0312875 }, "kuairand-pure": { "ndcg@10": 0.01020438066722946, "recall@20": 0.02944024263271866, "mrr@10": 0.015063492063492064, "hitrate@10": 0.04875, "coverage@20": 0.08303488526329753, "novelty@20": 8.948117664597367, "ild@20": 0.3961559210526317, "tail_share@20": 0 }, "kuairec": { "ndcg@10": 0.025199741324654856, "recall@20": 0.07359445159277848, "mrr@10": 0.04021956506538005, "hitrate@10": 0.15330396475770924, "coverage@20": 0.0757825370675453, "novelty@20": 10.696545615318078, "ild@20": 0.5682096046569161, "tail_share@20": 0.25616740088105727 }, "lastfm-1k": { "ndcg@10": 0.03420498950012055, "recall@20": 0.005536193059203976, "mrr@10": 0.08188828990715782, "hitrate@10": 0.19448476052249636, "coverage@20": 0.06620854636709039, "novelty@20": 11.785953522057493, "ild@20": 1, "tail_share@20": 0.04252539912917271 }, "mind-small": { "ndcg@10": 0.10438605537411773, "recall@20": 0.23435487629630433, "mrr@10": 0.11519305555555556, "hitrate@10": 0.2815, "coverage@20": 0.03530045809754783, "novelty@20": 9.11074070729466, "ild@20": 0.8690664473684211, "tail_share@20": 0 }, "ml-1m": { "ndcg@10": 0.16571785997436156, "recall@20": 0.08030891444054909, "mrr@10": 0.29925896123903056, "hitrate@10": 0.561525129982669, "coverage@20": 0.1904, "novelty@20": 8.78741203485964, "ild@20": 0.6610552457739166, "tail_share@20": 0.0001733102253032929 }, "ml-32m": { "ndcg@10": 0.0857927821154029, "recall@20": 0.05889227258038209, "mrr@10": 0.1680092634638089, "hitrate@10": 0.406865861411316, "coverage@20": 0.08543337151584575, "novelty@20": 10.61292646691966, "ild@20": 0.6287421299992135, "tail_share@20": 0.003655435473617292 }, "retailrocket": { "ndcg@10": 0.011910850954338106, "recall@20": 0.017241379310344827, "mrr@10": 0.015732758620689655, "hitrate@10": 0.034482758620689655, "coverage@20": 0.3200371057513915, "novelty@20": 10.923676524506241, "ild@20": 1, "tail_share@20": 0.49051724137931035 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 5, "compared_on_n_datasets": 10 }, { "rank": 17, "key": "ials", "system": "iALS", "index": 51.02, "family": "matrix-factorisation", "generative": false, "confidence": "reference", "origin": "Implicit-feedback alternating least squares", "citation": "Hu, Koren & Volinsky, ICDM 2008; tuning per Rendle et al., 'Revisiting the Performance of iALS', RecSys 2022.", "summary": "Alternating least squares over implicit feedback with confidence weighting (Hu, Koren & Volinsky, ICDM 2008). Rendle et al. (2022) reported that its published comparisons were undertuned, so it is tuned here rather than run at library defaults. A new user's factors can be folded in without re-solving the model.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 46.63, "cold_start": 46.56, "long_tail": 67.87, "beyond_accuracy": 70.66, "multi_objective": 35.49, "robustness": 46.11 }, "per_dataset_relevance": { "amazon-beauty-2023": 43.15217391304348, "goodreads-poetry": 40.326086956521735, "hm-fashion": 51.42857142857143, "kuairand-pure": 73.58695652173913, "kuairec": 3.2608695652173907, "lastfm-1k": 62.142857142857146, "mind-small": 23.369565217391305, "ml-1m": 19.673913043478258, "ml-32m": 54.673913043478265, "retailrocket": 94.67391304347825 }, "n_datasets": 10, "spread": 25.61965876702721, "monthly_cost_usd": 2.859877692874412, "train_cost_usd": 0.18488937880882328, "inference_cost_per_1k": 0.0002055608895056031, "latency_p50_ms": 0.13244350543970088, "latency_p95_ms": 0.13244350543970088, "model_mb": 44.5272576, "requires_gpu": false, "retrain_note": "full ALS re-solve; user factors can be folded in without a refit", "latency_caveat": "", "value_index": 128.4353220999625, "other_track_index": 50.99, "win_rate": 50.56799740344327, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.007548808509471626, "recall@20": 0.018191823258045265, "mrr@10": 0.008928472222222221, "hitrate@10": 0.0215, "coverage@20": 0.37355658198614317, "novelty@20": 11.81839281090302, "ild@20": 1, "tail_share@20": 0.011375 }, "goodreads-poetry": { "ndcg@10": 0.05686437652928566, "recall@20": 0.12219347253043128, "mrr@10": 0.08038323619378307, "hitrate@10": 0.17447916666666666, "coverage@20": 0.8952807440578712, "novelty@20": 11.360778796475023, "ild@20": 0.4768593236019738, "tail_share@20": 0.20716145833333333 }, "hm-fashion": { "ndcg@10": 0.006604990358683441, "recall@20": 0.011587117051981595, "mrr@10": 0.013577083333333335, "hitrate@10": 0.036, "coverage@20": 0.1435753770729633, "novelty@20": 12.56157231862647, "ild@20": 0.7075105316067941, "tail_share@20": 0.000075 }, "kuairand-pure": { "ndcg@10": 0.016567610824860482, "recall@20": 0.03970167369064441, "mrr@10": 0.024886507936507936, "hitrate@10": 0.06625, "coverage@20": 0.37193261705796526, "novelty@20": 10.681594784783575, "ild@20": 0.44315921052631596, "tail_share@20": 0.00065 }, "kuairec": { "ndcg@10": 0.011003769182410693, "recall@20": 0.01816831489144839, "mrr@10": 0.024214390602055796, "hitrate@10": 0.06343612334801763, "coverage@20": 0.7259747391543108, "novelty@20": 9.954719376294104, "ild@20": 0.8471971573731902, "tail_share@20": 0.0714977973568282 }, "lastfm-1k": { "ndcg@10": 0.053515420036759465, "recall@20": 0.009479067286985157, "mrr@10": 0.11744419102909669, "hitrate@10": 0.3134978229317852, "coverage@20": 0.047985365167221725, "novelty@20": 11.379927544009691, "ild@20": 1, "tail_share@20": 0.00007256894049346879 }, "mind-small": { "ndcg@10": 0.06359329382745764, "recall@20": 0.1534318200074779, "mrr@10": 0.07218531746031746, "hitrate@10": 0.18675, "coverage@20": 0.5580706009161951, "novelty@20": 10.308043551834928, "ild@20": 0.8885822368421054, "tail_share@20": 0.0149375 }, "ml-1m": { "ndcg@10": 0.11803415224166319, "recall@20": 0.06027701856270447, "mrr@10": 0.23977194575114852, "hitrate@10": 0.5069324090121318, "coverage@20": 0.49248, "novelty@20": 10.025289028073244, "ild@20": 0.7156737217900593, "tail_share@20": 0.00645580589254766 }, "ml-32m": { "ndcg@10": 0.09222174214573883, "recall@20": 0.05938929005990611, "mrr@10": 0.1853270466906831, "hitrate@10": 0.39351557533375714, "coverage@20": 0.23004963726613212, "novelty@20": 10.629177937784613, "ild@20": 0.6809259552017244, "tail_share@20": 0.0003814367450731087 }, "retailrocket": { "ndcg@10": 0.026771809232232817, "recall@20": 0.0596264367816092, "mrr@10": 0.02730911330049261, "hitrate@10": 0.06896551724137931, "coverage@20": 0.5700371057513914, "novelty@20": 11.419734170969711, "ild@20": 1, "tail_share@20": 0.6831896551724138 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 2, "compared_on_n_datasets": 8 }, { "rank": 18, "key": "ease_lambdamart", "system": "EASEᴿ + LambdaMART", "index": 50.88, "family": "multi-stage", "generative": false, "confidence": "reference", "origin": "Linear autoencoder retrieval with a listwise gradient-boosted ranker", "citation": "Steck, WWW 2019 (retrieval); Burges, 'From RankNet to LambdaRank to LambdaMART', MSR-TR-2010-82 (ranking).", "summary": "A closed-form linear autoencoder produces the candidates and a listwise gradient-boosted ranker orders them. It uses no GPU, no embeddings and no training loop, and it inherits the autoencoder's catalogue-size limit.", "stages": "retrieval + ranking", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": "ease", "retriever_label": null, "ranker_label": "LambdaMART ranker", "is_new": false, "dimensions": { "relevance": 45.96, "cold_start": 44.6, "long_tail": 58.63, "beyond_accuracy": 66.32, "multi_objective": 50.8, "robustness": 50.61 }, "per_dataset_relevance": { "amazon-beauty-2023": 25.217391304347824, "goodreads-poetry": 50.434782608695656, "kuairand-pure": 53.15217391304348, "kuairec": 55.97826086956522, "mind-small": 39.78260869565217, "ml-1m": 43.69565217391305, "ml-32m": 64.23913043478261, "retailrocket": 35.21739130434783 }, "n_datasets": 8, "spread": 11.700373509116517, "monthly_cost_usd": 76.18868189871066, "train_cost_usd": 0.016785005840780392, "inference_cost_per_1k": 0.007611566712330325, "latency_p50_ms": 3.8733800368326734, "latency_p95_ms": 3.8733800368326734, "model_mb": 456.86672749999997, "requires_gpu": false, "retrain_note": "retriever refit on train+valid for serving; ranker weights carried over from the valid-window fit", "latency_caveat": "includes constructing the ranker's features online, in Python, for every request — work a deployed pipeline precomputes or caches, so this is an upper bound", "value_index": 4.807902791610168, "other_track_index": 50.26, "win_rate": 48.19264433597135, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.0043700062684608925, "recall@20": 0.014963345929242125, "mrr@10": 0.005717063492063492, "hitrate@10": 0.0165, "coverage@20": 0.31928406466512704, "novelty@20": 11.731525169255445, "ild@20": 1, "tail_share@20": 0.032925 }, "goodreads-poetry": { "ndcg@10": 0.06171544728927122, "recall@20": 0.14840940564775731, "mrr@10": 0.07868096891534392, "hitrate@10": 0.20768229166666666, "coverage@20": 0.5053393041681019, "novelty@20": 9.449008816706586, "ild@20": 0.4890235231633773, "tail_share@20": 0.0525390625 }, "kuairand-pure": { "ndcg@10": 0.013097507249697103, "recall@20": 0.03559307124387542, "mrr@10": 0.02175297619047619, "hitrate@10": 0.06225, "coverage@20": 0.34314895874784457, "novelty@20": 10.557511377264307, "ild@20": 0.42789539473684224, "tail_share@20": 0.0044625 }, "kuairec": { "ndcg@10": 0.047516735554230666, "recall@20": 0.08101346255560857, "mrr@10": 0.0941119502132718, "hitrate@10": 0.2211453744493392, "coverage@20": 0.16254805052169138, "novelty@20": 8.789841486122066, "ild@20": 0.8023303833072314, "tail_share@20": 0.007136563876651982 }, "mind-small": { "ndcg@10": 0.09503936882658037, "recall@20": 0.23991488218631213, "mrr@10": 0.10425386904761905, "hitrate@10": 0.26825, "coverage@20": 0.21126381029372138, "novelty@20": 9.431086077625977, "ild@20": 0.9245730263157894, "tail_share@20": 0.0025375 }, "ml-1m": { "ndcg@10": 0.14617610042627674, "recall@20": 0.07437168674133546, "mrr@10": 0.27067859206074113, "hitrate@10": 0.5632582322357019, "coverage@20": 0.29344, "novelty@20": 9.200042787074002, "ild@20": 0.7230914755716723, "tail_share@20": 0.0004332755632582322 }, "ml-32m": { "ndcg@10": 0.09525083118002384, "recall@20": 0.06595596591835065, "mrr@10": 0.18839317248408158, "hitrate@10": 0.4380165289256198, "coverage@20": 0.2014127529591447, "novelty@20": 10.768864578341173, "ild@20": 0.6772534766994034, "tail_share@20": 0.003337571519389701 }, "retailrocket": { "ndcg@10": 0.006733169078774423, "recall@20": 0.029094827586206896, "mrr@10": 0.007697044334975369, "hitrate@10": 0.02586206896551724, "coverage@20": 0.24257884972170687, "novelty@20": 10.486138956379229, "ild@20": 1, "tail_share@20": 0.33017241379310347 } }, "beats_next_significantly": null, "beats_next_on_n_datasets": null, "compared_on_n_datasets": null }, { "rank": 19, "key": "option_full", "system": "Option-Attention Scorer (full catalogue)", "index": 50.31, "family": "option-scoring", "generative": false, "confidence": "reproduction", "origin": "Target-conditioned attention over a user state, scored in one parallel pass", "citation": "Zhou et al., 'Deep Interest Network for Click-Through Rate Prediction', KDD 2018 (target-conditioned attention); Zaratiana et al., 'GLiNER2', arXiv:2507.18546 (schema-conditioned one-pass scoring).", "summary": "Scores every catalogue item in one pass, with no retrieval stage and no candidate set. Each item forms its own query over the user's recent history and receives its own attended view of it, so the user representation depends on the item being scored — unlike a two-tower model, where one user vector is dotted with every item. The user's state is read from their history at request time and an item's vector combines its content with its ID, so a new interaction or a newly added item changes the output without retraining. The attention runs once per item per request, where a dot product runs once.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": "v3.2", "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": true, "dimensions": { "relevance": 46.78, "cold_start": 47.78, "long_tail": 55.47, "beyond_accuracy": 48.19, "multi_objective": 63.75, "robustness": 48.36 }, "per_dataset_relevance": { "amazon-beauty-2023": 41.30434782608695, "goodreads-poetry": 48.91304347826087, "hm-fashion": 31.54761904761905, "kuairand-pure": 80.54347826086956, "kuairec": 30.543478260869563, "lastfm-1k": 45.11904761904763, "mind-small": 74.67391304347827, "ml-1m": 25.543478260869566, "ml-32m": 43.15217391304348, "retailrocket": 46.46739130434783 }, "n_datasets": 10, "spread": 17.074053462407, "monthly_cost_usd": 5.947650964034865, "train_cost_usd": 0.178101596255598, "inference_cost_per_1k": 0.0005172909020323014, "latency_p50_ms": 0.2596416337609584, "latency_p95_ms": 0.2786000321000081, "model_mb": 16.948072, "requires_gpu": true, "retrain_note": "full retrain, or warm-start continuation from the last checkpoint", "latency_caveat": "", "value_index": 60.90157991231959, "other_track_index": 32.61, "win_rate": 45.84202717114361, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.007243294091116154, "recall@20": 0.016843951317044536, "mrr@10": 0.010011904761904764, "hitrate@10": 0.02225, "coverage@20": 0.05484988452655889, "novelty@20": 10.116486550605266, "ild@20": 1, "tail_share@20": 0.00265 }, "goodreads-poetry": { "ndcg@10": 0.06308252019023752, "recall@20": 0.126578937092114, "mrr@10": 0.0856424644510582, "hitrate@10": 0.19401041666666666, "coverage@20": 0.5959352394075095, "novelty@20": 9.28839133128975, "ild@20": 0.48668662623355274, "tail_share@20": 0.06751302083333334 }, "hm-fashion": { "ndcg@10": 0.0034191030199448038, "recall@20": 0.007527440647111448, "mrr@10": 0.006735615079365079, "hitrate@10": 0.0195, "coverage@20": 0.2952299957477676, "novelty@20": 13.375797920592701, "ild@20": 0.5032544237789153, "tail_share@20": 0.08525 }, "kuairand-pure": { "ndcg@10": 0.0166054602745283, "recall@20": 0.03824256998666404, "mrr@10": 0.028099801587301586, "hitrate@10": 0.076, "coverage@20": 0.06963788300835655, "novelty@20": 9.044739257462178, "ild@20": 0.4152769736842106, "tail_share@20": 0.000575 }, "kuairec": { "ndcg@10": 0.03703181393861229, "recall@20": 0.05938030885951125, "mrr@10": 0.08298195930354521, "hitrate@10": 0.19647577092511012, "coverage@20": 0.23668314113124655, "novelty@20": 9.561586719423138, "ild@20": 0.5887365842622554, "tail_share@20": 0.07933920704845815 }, "lastfm-1k": { "ndcg@10": 0.03149935228572283, "recall@20": 0.006251737277986909, "mrr@10": 0.08122077084341237, "hitrate@10": 0.21770682148040638, "coverage@20": 0.07054739903372578, "novelty@20": 11.91525440958361, "ild@20": 1, "tail_share@20": 0.027285921625544267 }, "mind-small": { "ndcg@10": 0.13101718775993612, "recall@20": 0.291672688258379, "mrr@10": 0.14634375, "hitrate@10": 0.33275, "coverage@20": 0.18916734033953111, "novelty@20": 8.678920286972057, "ild@20": 0.9023967105263156, "tail_share@20": 0.001025 }, "ml-1m": { "ndcg@10": 0.12088657243635628, "recall@20": 0.06365096575941859, "mrr@10": 0.23656880140848943, "hitrate@10": 0.4930675909878683, "coverage@20": 0.40352, "novelty@20": 9.79254025563684, "ild@20": 0.5825868145565919, "tail_share@20": 0.012348353552859619 }, "ml-32m": { "ndcg@10": 0.08571261303772884, "recall@20": 0.054146218381422244, "mrr@10": 0.17008547008547012, "hitrate@10": 0.38397965670692946, "coverage@20": 0.10414280259641084, "novelty@20": 10.449129104096158, "ild@20": 0.6257555631258143, "tail_share@20": 0.007501589319771138 }, "retailrocket": { "ndcg@10": 0.0120844811894619, "recall@20": 0.021551724137931036, "mrr@10": 0.01293103448275862, "hitrate@10": 0.02586206896551724, "coverage@20": 0.4735621521335807, "novelty@20": 10.696961102481247, "ild@20": 1, "tail_share@20": 0.38620689655172413 } }, "beats_next_significantly": null, "beats_next_on_n_datasets": null, "compared_on_n_datasets": null }, { "rank": 20, "key": "hstu", "system": "HSTU (Generative Recommenders)", "index": 49.65, "family": "generative-sequential", "generative": true, "confidence": "reproduction", "origin": "Meta — Hierarchical Sequential Transduction Units", "citation": "Zhai et al., 'Actions Speak Louder than Words: Trillion-Parameter Sequential Transducers for Generative Recommendations', ICML 2024 (arXiv:2402.17152); reference code at github.com/meta-recsys/generative-recommenders.", "summary": "Recommendation reformulated as sequential transduction (Zhai et al., ICML 2024). The unit replaces softmax attention with a pointwise SiLU gate and adds a relative attention bias over position and elapsed time, which suits a vocabulary that changes continuously. Its authors deploy it at very large scale; what runs here is the published architecture at benchmark scale, so the result speaks to the design rather than to the scale.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 40.8, "cold_start": 47.7, "long_tail": 59.95, "beyond_accuracy": 63.76, "multi_objective": 68.22, "robustness": 40.14 }, "per_dataset_relevance": { "amazon-beauty-2023": 20, "goodreads-poetry": 16.304347826086957, "hm-fashion": 29.642857142857146, "kuairand-pure": 72.93478260869566, "kuairec": 28.47826086956522, "lastfm-1k": 23.21428571428571, "mind-small": 71.73913043478261, "ml-1m": 13.586956521739129, "ml-32m": 86.30434782608695, "retailrocket": 45.76086956521739 }, "n_datasets": 10, "spread": 25.384879310793433, "monthly_cost_usd": 7.626565181503656, "train_cost_usd": 1.3664659172184852, "inference_cost_per_1k": 0.00016824384416032448, "latency_p50_ms": 0.10210678781674805, "latency_p95_ms": 0.180053563858156, "model_mb": 8.013344, "requires_gpu": true, "retrain_note": "tokenizer refit when the catalogue shifts, then a model retrain; the tokenizer is the harder half because changing item codes invalidates the model", "latency_caveat": "", "value_index": 46.86908602589658, "other_track_index": 34.79, "win_rate": 45.106528492606834, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.0032981664432874245, "recall@20": 0.008661512420410725, "mrr@10": 0.0041900793650793645, "hitrate@10": 0.01125, "coverage@20": 0.1411342057993328, "novelty@20": 10.65105703900371, "ild@20": 1, "tail_share@20": 0.0013375 }, "goodreads-poetry": { "ndcg@10": 0.03926248061987808, "recall@20": 0.07736692229313376, "mrr@10": 0.054652622767857144, "hitrate@10": 0.11588541666666667, "coverage@20": 0.9121598346538065, "novelty@20": 10.371357162381594, "ild@20": 0.5063292386239037, "tail_share@20": 0.16927083333333334 }, "hm-fashion": { "ndcg@10": 0.003169243109474735, "recall@20": 0.005895737604680683, "mrr@10": 0.006979563492063493, "hitrate@10": 0.01775, "coverage@20": 0.22419270116811327, "novelty@20": 12.654365811600814, "ild@20": 0.8352234147571448, "tail_share@20": 0.0165125 }, "kuairand-pure": { "ndcg@10": 0.015429249448348082, "recall@20": 0.040614098731082834, "mrr@10": 0.02330952380952381, "hitrate@10": 0.07425, "coverage@20": 0.11924658442764292, "novelty@20": 9.774269511471886, "ild@20": 0.44745460526315806, "tail_share@20": 0.000125 }, "kuairec": { "ndcg@10": 0.03773092615926403, "recall@20": 0.05420016605025584, "mrr@10": 0.08152576742885112, "hitrate@10": 0.1920704845814978, "coverage@20": 0.4673256452498627, "novelty@20": 10.050692458522201, "ild@20": 0.6652583284044895, "tail_share@20": 0.1463876651982379 }, "lastfm-1k": { "ndcg@10": 0.013192091567221008, "recall@20": 0.0018723989143490498, "mrr@10": 0.03972458359250812, "hitrate@10": 0.10885341074020319, "coverage@20": 0.1286411182513251, "novelty@20": 14.104516162920303, "ild@20": 1, "tail_share@20": 0.1362844702467344 }, "mind-small": { "ndcg@10": 0.12733056999086517, "recall@20": 0.308296178090915, "mrr@10": 0.13909464285714285, "hitrate@10": 0.32825, "coverage@20": 0.14201023982753974, "novelty@20": 8.573348963720772, "ild@20": 0.9102236842105264, "tail_share@20": 0.0000375 }, "ml-1m": { "ndcg@10": 0.10768043915663982, "recall@20": 0.06083313826550839, "mrr@10": 0.22039008555473025, "hitrate@10": 0.47227036395147315, "coverage@20": 0.632, "novelty@20": 10.32097057796451, "ild@20": 0.6202607288714467, "tail_share@20": 0.05177642980935875 }, "ml-32m": { "ndcg@10": 0.10604617630250883, "recall@20": 0.0707317632690064, "mrr@10": 0.21669340760249853, "hitrate@10": 0.45772409408773046, "coverage@20": 0.13707521954944635, "novelty@20": 11.585624997124619, "ild@20": 0.666051571072025, "tail_share@20": 0.004799745708836618 }, "retailrocket": { "ndcg@10": 0.012256667323697108, "recall@20": 0.02586206896551724, "mrr@10": 0.010344827586206896, "hitrate@10": 0.017241379310344827, "coverage@20": 0.5468460111317254, "novelty@20": 10.86175886257779, "ild@20": 1, "tail_share@20": 0.4547413793103448 } }, "beats_next_significantly": null, "beats_next_on_n_datasets": null, "compared_on_n_datasets": null }, { "rank": 21, "key": "option_calibrated", "system": "Calibrated Multi-Outcome Option Scorer", "index": 47.55, "family": "option-scoring", "generative": false, "confidence": "reproduction", "origin": "Calibrated per-outcome probabilities from one forward pass", "citation": "Kumar, Sarawagi & Jain, 'Trainable Calibration Measures from Kernel Mean Embeddings', ICML 2018; Platt, 'Probabilistic Outputs for Support Vector Machines', 1999; Guo et al., 'On Calibration of Modern Neural Networks', ICML 2017; Ma et al., KDD 2018 (multi-outcome heads).", "summary": "A one-pass option scorer with one output head per logged engagement signal, returning a probability per outcome per item rather than a single relevance score. A kernel calibration penalty is applied during training and a Platt scaling is fitted afterwards on held-out data; the calibration error before and after is recorded against a stated population. Slates are ordered by fixed signal weights chosen before any result was measured. On a dataset that logs a single signal it runs one head and is reported as such.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": "v3.2", "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": true, "dimensions": { "relevance": 41.79, "cold_start": 45.42, "long_tail": 53.56, "beyond_accuracy": 51.43, "multi_objective": 62.81, "robustness": 44.36 }, "per_dataset_relevance": { "amazon-beauty-2023": 43.36956521739131, "goodreads-poetry": 53.47826086956522, "hm-fashion": 34.404761904761905, "kuairand-pure": 57.934782608695656, "kuairec": 42.934782608695656, "lastfm-1k": 42.50000000000001, "mind-small": 63.369565217391305, "ml-1m": 20, "ml-32m": 50.21739130434783, "retailrocket": 9.673913043478262 }, "n_datasets": 10, "spread": 15.808436085211149, "monthly_cost_usd": 6.804629933799783, "train_cost_usd": 0.3743239672501636, "inference_cost_per_1k": 0.0005176320676261571, "latency_p50_ms": 0.2590089724144038, "latency_p95_ms": 0.2786863831228379, "model_mb": 16.9483048, "requires_gpu": true, "retrain_note": "full retrain, or warm-start continuation from the last checkpoint", "latency_caveat": "", "value_index": 50.31392482261607, "other_track_index": 30.94, "win_rate": 42.48214865303696, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.006627882916288606, "recall@20": 0.01863864228251397, "mrr@10": 0.009184623015873016, "hitrate@10": 0.0235, "coverage@20": 0.0558763151141904, "novelty@20": 10.12154430574612, "ild@20": 1, "tail_share@20": 0.002075 }, "goodreads-poetry": { "ndcg@10": 0.06549798991315328, "recall@20": 0.1292478235661975, "mrr@10": 0.08871941137566138, "hitrate@10": 0.1953125, "coverage@20": 0.6310713055459869, "novelty@20": 9.341257803867672, "ild@20": 0.48142475328947376, "tail_share@20": 0.07086588541666666 }, "hm-fashion": { "ndcg@10": 0.0038824928080265593, "recall@20": 0.0073513572254140305, "mrr@10": 0.007932638888888888, "hitrate@10": 0.0205, "coverage@20": 0.29548012706671, "novelty@20": 13.369206206109666, "ild@20": 0.5085578048844034, "tail_share@20": 0.0860125 }, "kuairand-pure": { "ndcg@10": 0.014031346844359584, "recall@20": 0.03651497425388069, "mrr@10": 0.021852678571428568, "hitrate@10": 0.06675, "coverage@20": 0.0741477649555644, "novelty@20": 9.422367982157713, "ild@20": 0.45053815789473695, "tail_share@20": 0.0003875 }, "kuairec": { "ndcg@10": 0.045000869343417606, "recall@20": 0.06213327364496536, "mrr@10": 0.09784910146143626, "hitrate@10": 0.21762114537444935, "coverage@20": 0.25370675453047775, "novelty@20": 9.474485429998452, "ild@20": 0.613192197787989, "tail_share@20": 0.04458149779735683 }, "lastfm-1k": { "ndcg@10": 0.028413130974520584, "recall@20": 0.0058575093538093745, "mrr@10": 0.0694882161863294, "hitrate@10": 0.19738751814223512, "coverage@20": 0.0663961724283503, "novelty@20": 11.841822168545404, "ild@20": 1, "tail_share@20": 0.02119013062409289 }, "mind-small": { "ndcg@10": 0.12329160115494475, "recall@20": 0.2816854294645412, "mrr@10": 0.13598740079365082, "hitrate@10": 0.316, "coverage@20": 0.22096469954190245, "novelty@20": 8.65790917018205, "ild@20": 0.9087519736842105, "tail_share@20": 0.001275 }, "ml-1m": { "ndcg@10": 0.11639626084577111, "recall@20": 0.06336209430942077, "mrr@10": 0.2211472861819482, "hitrate@10": 0.4904679376083189, "coverage@20": 0.40032, "novelty@20": 9.814820281329311, "ild@20": 0.5843277396832502, "tail_share@20": 0.011611785095320624 }, "ml-32m": { "ndcg@10": 0.08716014338206457, "recall@20": 0.05686196013184756, "mrr@10": 0.17278529551256822, "hitrate@10": 0.4043229497774952, "coverage@20": 0.1034746086292478, "novelty@20": 10.477586070096274, "ild@20": 0.6222918745666545, "tail_share@20": 0.003909726636999364 }, "retailrocket": { "ndcg@10": 0.002719524799876972, "recall@20": 0.011494252873563218, "mrr@10": 0.0010775862068965517, "hitrate@10": 0.008620689655172414, "coverage@20": 0.4837662337662338, "novelty@20": 10.673689698031854, "ild@20": 1, "tail_share@20": 0.36982758620689654 } }, "beats_next_significantly": null, "beats_next_on_n_datasets": null, "compared_on_n_datasets": null }, { "rank": 22, "key": "netflix_foundation_style", "system": "Netflix-style foundation model", "index": 45.97, "family": "foundation-model", "generative": true, "confidence": "proxy", "origin": "Netflix — Foundation Model for Personalized Recommendation", "citation": "Netflix Technology Blog, 'Foundation Model for Personalized Recommendation', March 2025; Netflix Personalization, Search and Recommendation workshop, 2025.", "summary": "One autoregressive model over a user's entire interaction history, trained to predict several future interactions at once and read out from several layers. An item's representation combines a learned ID embedding with a projection of its metadata, so an item with no interaction history still has a representation.", "stages": "single-stage", "proxy_of": "Netflix foundation model for recommendations", "proxy_caveat": "Netflix's model is trained on their own interaction logs at a scale and context length not reproducible here, and its published description omits most implementation detail. This entrant implements the four design choices Netflix stated publicly — unified autoregressive user model, multi-token prediction, multi-layer readout, content-aware item embeddings — at benchmark scale.", "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 34.24, "cold_start": 40.64, "long_tail": 66.09, "beyond_accuracy": 66.61, "multi_objective": 56.92, "robustness": 37.51 }, "per_dataset_relevance": { "amazon-beauty-2023": 19.67391304347826, "goodreads-poetry": 11.630434782608697, "hm-fashion": 41.07142857142857, "kuairand-pure": 92.93478260869566, "kuairec": 14.021739130434781, "lastfm-1k": 38.69047619047619, "mind-small": 34.78260869565217, "ml-1m": 41.52173913043479, "ml-32m": 20, "retailrocket": 28.043478260869563 }, "n_datasets": 10, "spread": 22.222030972561978, "monthly_cost_usd": 3.6972098839075898, "train_cost_usd": 0.22413558936010713, "inference_cost_per_1k": 0.00027222200701911237, "latency_p50_ms": 0.157589356362378, "latency_p95_ms": 0.19649461547944208, "model_mb": 18.263072, "requires_gpu": true, "retrain_note": "tokenizer refit when the catalogue shifts, then a model retrain; the tokenizer is the harder half because changing item codes invalidates the model", "latency_caveat": "", "value_index": 89.52738208394153, "other_track_index": 26.65, "win_rate": 42.41665507488518, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.00317932858246893, "recall@20": 0.01033782402660038, "mrr@10": 0.0039245039682539685, "hitrate@10": 0.012, "coverage@20": 0.203810623556582, "novelty@20": 11.925501399307954, "ild@20": 1, "tail_share@20": 0.0697125 }, "goodreads-poetry": { "ndcg@10": 0.032304815626998294, "recall@20": 0.0781371420114605, "mrr@10": 0.040885933366402116, "hitrate@10": 0.11002604166666667, "coverage@20": 0.63761625904237, "novelty@20": 10.228380165099983, "ild@20": 0.49113084224232467, "tail_share@20": 0.18056640625 }, "hm-fashion": { "ndcg@10": 0.004487840874978534, "recall@20": 0.009042980459726745, "mrr@10": 0.009682539682539683, "hitrate@10": 0.02875, "coverage@20": 0.2552590109807649, "novelty@20": 14.297974314408606, "ild@20": 0.6624270720694095, "tail_share@20": 0.1086125 }, "kuairand-pure": { "ndcg@10": 0.01756128723650169, "recall@20": 0.0472374848598992, "mrr@10": 0.025334821428571432, "hitrate@10": 0.082, "coverage@20": 0.16540655259318213, "novelty@20": 9.904676001611348, "ild@20": 0.44644671052631596, "tail_share@20": 0.0007375 }, "kuairec": { "ndcg@10": 0.024277345023105194, "recall@20": 0.04027915557158624, "mrr@10": 0.046323683658485416, "hitrate@10": 0.12070484581497798, "coverage@20": 0.5107084019769358, "novelty@20": 10.157868617651884, "ild@20": 0.7863972678434379, "tail_share@20": 0.1986784140969163 }, "lastfm-1k": { "ndcg@10": 0.027621255561021556, "recall@20": 0.0026387804821539104, "mrr@10": 0.07094362660400395, "hitrate@10": 0.1683599419448476, "coverage@20": 0.10363994558844224, "novelty@20": 13.54011451341908, "ild@20": 1, "tail_share@20": 0.06269956458635705 }, "mind-small": { "ndcg@10": 0.08342046283070279, "recall@20": 0.2253375620906871, "mrr@10": 0.09116984126984128, "hitrate@10": 0.2455, "coverage@20": 0.34384263001886284, "novelty@20": 9.347220168931011, "ild@20": 0.8796131578947369, "tail_share@20": 0.0130125 }, "ml-1m": { "ndcg@10": 0.14084919060222872, "recall@20": 0.07908176068663492, "mrr@10": 0.26700537811889635, "hitrate@10": 0.5259965337954939, "coverage@20": 0.60032, "novelty@20": 10.189008505817313, "ild@20": 0.6012633088336328, "tail_share@20": 0.050173310225303296 }, "ml-32m": { "ndcg@10": 0.06194211452631475, "recall@20": 0.03830816595196603, "mrr@10": 0.12934792480247026, "hitrate@10": 0.3038779402415766, "coverage@20": 0.0220504009163803, "novelty@20": 9.035083661318387, "ild@20": 0.7373771367983802, "tail_share@20": 0 }, "retailrocket": { "ndcg@10": 0.00854946890661663, "recall@20": 0.015804597701149423, "mrr@10": 0.006773399014778325, "hitrate@10": 0.02586206896551724, "coverage@20": 0.40166975881261596, "novelty@20": 11.285703421781546, "ild@20": 1, "tail_share@20": 0.6521551724137931 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 4, "compared_on_n_datasets": 10 }, { "rank": 23, "key": "onerec_style", "system": "OneRec-style session generator", "index": 42.37, "family": "generative-sequential", "generative": true, "confidence": "proxy", "origin": "Kuaishou — OneRec", "citation": "Deng et al., 'OneRec Technical Report', arXiv:2506.13695; 'OneRec-V2', arXiv:2508.20900.", "summary": "One model generates a whole session at once in place of separate retrieval, ranking and reranking stages, and is then aligned to a reward model rather than to logged clicks. Because the slate is decoded jointly, an item's own relevance can be traded against the coherence of the slate, which a pipeline of independently scored items cannot do.", "stages": "single-stage (session-wise generation + preference alignment)", "proxy_of": "OneRec (Kuaishou)", "proxy_caveat": "OneRec is a production system trained on Kuaishou's logs with a multimodal tokenizer, a mixture-of-experts decoder at a scale far beyond anything here, and a reward model built from proprietary engagement data. This entrant implements session-wise generation and preference alignment at benchmark scale on public data.", "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 33.87, "cold_start": 41.28, "long_tail": 66.41, "beyond_accuracy": 53.23, "multi_objective": 41.85, "robustness": 32.34 }, "per_dataset_relevance": { "amazon-beauty-2023": 17.499999999999996, "goodreads-poetry": 9.456521739130434, "hm-fashion": 39.76190476190476, "kuairand-pure": 22.282608695652172, "kuairec": 69.02173913043478, "lastfm-1k": 57.26190476190476, "mind-small": 14.67391304347826, "ml-1m": 18.804347826086957, "ml-32m": 35.43478260869565, "retailrocket": 54.45652173913044 }, "n_datasets": 10, "spread": 19.589588598230023, "monthly_cost_usd": 115.16099075686061, "train_cost_usd": 0.46215251876127594, "inference_cost_per_1k": 0.011315062730024906, "latency_p50_ms": 6.152706555126851, "latency_p95_ms": 6.152706555126851, "model_mb": 9.413252, "requires_gpu": true, "retrain_note": "tokenizer refit when the catalogue shifts, then a model retrain; the tokenizer is the harder half because changing item codes invalidates the model", "latency_caveat": "", "value_index": 2.6487320555603566, "other_track_index": 28.82, "win_rate": 38.28766170538542, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.0016514739896686193, "recall@20": 0.014105892521221469, "mrr@10": 0.0017742063492063491, "hitrate@10": 0.00675, "coverage@20": 0.47780343854246854, "novelty@20": 12.15759970458475, "ild@20": 1, "tail_share@20": 0.1758125 }, "goodreads-poetry": { "ndcg@10": 0.028119216305702938, "recall@20": 0.08158617346377033, "mrr@10": 0.03922164351851851, "hitrate@10": 0.10481770833333333, "coverage@20": 0.7061660351360661, "novelty@20": 9.464550564108169, "ild@20": 0.51032929002193, "tail_share@20": 0.11526692708333333 }, "hm-fashion": { "ndcg@10": 0.004012639956827644, "recall@20": 0.010726215070127483, "mrr@10": 0.007707242063492063, "hitrate@10": 0.02625, "coverage@20": 0.09202331223892543, "novelty@20": 12.170250620936088, "ild@20": 0.6490271968233446, "tail_share@20": 0.0806125 }, "kuairand-pure": { "ndcg@10": 0.010401254878657003, "recall@20": 0.02229587880331281, "mrr@10": 0.015172519841269841, "hitrate@10": 0.0515, "coverage@20": 0.08608568775699695, "novelty@20": 8.753616523178303, "ild@20": 0.4344394736842107, "tail_share@20": 0.0050625 }, "kuairec": { "ndcg@10": 0.05031381716439708, "recall@20": 0.08693619353102285, "mrr@10": 0.08844731137682679, "hitrate@10": 0.2696035242290749, "coverage@20": 0.2613948380010983, "novelty@20": 8.53758163188539, "ild@20": 0.6903838409284717, "tail_share@20": 0.034229074889867844 }, "lastfm-1k": { "ndcg@10": 0.03604613457205718, "recall@20": 0.010018442007086617, "mrr@10": 0.07418158361554589, "hitrate@10": 0.20319303338171263, "coverage@20": 0.07068811857967072, "novelty@20": 12.226999238821906, "ild@20": 1, "tail_share@20": 0.10275761973875182 }, "mind-small": { "ndcg@10": 0.04670419644229557, "recall@20": 0.15003505034052633, "mrr@10": 0.05014156746031746, "hitrate@10": 0.1525, "coverage@20": 0.5507949339800593, "novelty@20": 9.731631278623773, "ild@20": 0.8611269736842105, "tail_share@20": 0.0624 }, "ml-1m": { "ndcg@10": 0.11077208808267386, "recall@20": 0.07188577788148458, "mrr@10": 0.19542653572116306, "hitrate@10": 0.49913344887348354, "coverage@20": 0.29568, "novelty@20": 8.994056080984763, "ild@20": 0.7000759099764681, "tail_share@20": 0.016637781629116118 }, "ml-32m": { "ndcg@10": 0.07496737475319083, "recall@20": 0.040961524334478194, "mrr@10": 0.1667105621651076, "hitrate@10": 0.3477431659249841, "coverage@20": 0.04028255059182894, "novelty@20": 8.65132700875458, "ild@20": 0.7248660266224063, "tail_share@20": 0.0026064844246662428 }, "retailrocket": { "ndcg@10": 0.01293103448275862, "recall@20": 0.02783764367816092, "mrr@10": 0.013218390804597701, "hitrate@10": 0.02586206896551724, "coverage@20": 0.21150278293135436, "novelty@20": 9.256094923115116, "ild@20": 1, "tail_share@20": 0.14698275862068966 } }, "beats_next_significantly": null, "beats_next_on_n_datasets": null, "compared_on_n_datasets": null }, { "rank": 24, "key": "option_rerank", "system": "ItemKNN + Option-Attention reranker", "index": 40.17, "family": "multi-stage-neural", "generative": false, "confidence": "reproduction", "origin": "Parallel option scoring used as the ranking stage of an existing pipeline", "citation": "Zhou et al., KDD 2018 (target attention); Covington, Adams & Sargin, RecSys 2016 (the stage this replaces).", "summary": "A neighbourhood model produces the candidates, and the ranking stage scores all of them in one parallel pass against the user's latest history, returning a probability per candidate normalised across the presented set. It uses the same retriever, the same retriever parameters and the same 200-candidate budget as the other neighbourhood pipelines in this benchmark, so the ranking stage is the only thing that differs between them.", "stages": "retrieval + option-attention ranking", "proxy_of": null, "proxy_caveat": null, "introduced_in": "v3.2", "native_retriever": "itemknn", "retriever_label": null, "ranker_label": "Option-Attention reranker", "is_new": true, "dimensions": { "relevance": 30.61, "cold_start": 38.07, "long_tail": 52.05, "beyond_accuracy": 69.87, "multi_objective": 38.79, "robustness": 33.78 }, "per_dataset_relevance": { "amazon-beauty-2023": 18.369565217391305, "goodreads-poetry": 25.326086956521735, "hm-fashion": 42.85714285714286, "kuairand-pure": 18.369565217391305, "kuairec": 24.347826086956523, "lastfm-1k": 55.476190476190474, "mind-small": 36.19565217391305, "ml-1m": 29.89130434782609, "ml-32m": 27.282608695652172, "retailrocket": 27.989130434782606 }, "n_datasets": 10, "spread": 10.884702589918676, "monthly_cost_usd": 6.334577594689767, "train_cost_usd": 0.054131717433350476, "inference_cost_per_1k": 0.0006099104623854692, "latency_p50_ms": 0.3301160681985026, "latency_p95_ms": 0.34653893566253935, "model_mb": 47.568256000000005, "requires_gpu": true, "retrain_note": "retriever refit on train+valid for serving; option head carried over from the valid-window fit", "latency_caveat": "includes constructing the ranker's features online, in Python, for every request — work a deployed pipeline precomputes or caches, so this is an upper bound", "value_index": 45.653373170202585, "other_track_index": 38.52, "win_rate": 36.2318588584432, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.002901487000184543, "recall@20": 0.009353905527862524, "mrr@10": 0.0034146825396825396, "hitrate@10": 0.01125, "coverage@20": 0.48729792147806006, "novelty@20": 13.523423076884196, "ild@20": 1, "tail_share@20": 0.40945 }, "goodreads-poetry": { "ndcg@10": 0.04271781966457821, "recall@20": 0.10329570019472435, "mrr@10": 0.05894484747023809, "hitrate@10": 0.14518229166666666, "coverage@20": 0.4516017912504306, "novelty@20": 9.675196458085454, "ild@20": 0.5105819969846492, "tail_share@20": 0.05849609375 }, "hm-fashion": { "ndcg@10": 0.004582764196301507, "recall@20": 0.01059296101607334, "mrr@10": 0.009842063492063492, "hitrate@10": 0.026, "coverage@20": 0.2185897596238025, "novelty@20": 12.552616949940909, "ild@20": 0.7972259283922594, "tail_share@20": 0.057675 }, "kuairand-pure": { "ndcg@10": 0.009038373925650557, "recall@20": 0.027780381779887925, "mrr@10": 0.013519940476190475, "hitrate@10": 0.04275, "coverage@20": 0.1918026263430163, "novelty@20": 9.935888498212504, "ild@20": 0.45589868421052643, "tail_share@20": 0.001325 }, "kuairec": { "ndcg@10": 0.03140163828925673, "recall@20": 0.053136254848413465, "mrr@10": 0.06298930144745121, "hitrate@10": 0.17885462555066078, "coverage@20": 0.12081274025260846, "novelty@20": 8.903370778819468, "ild@20": 0.8001138822315024, "tail_share@20": 0 }, "lastfm-1k": { "ndcg@10": 0.04842940509337828, "recall@20": 0.0064969182002620985, "mrr@10": 0.10875550026493423, "hitrate@10": 0.25834542815674894, "coverage@20": 0.016276560814297104, "novelty@20": 11.036400695397754, "ild@20": 1, "tail_share@20": 0 }, "mind-small": { "ndcg@10": 0.09272384774413726, "recall@20": 0.21089444472903257, "mrr@10": 0.10633561507936508, "hitrate@10": 0.2495, "coverage@20": 0.636755591484775, "novelty@20": 9.9568269624586, "ild@20": 0.9073638157894737, "tail_share@20": 0.0510125 }, "ml-1m": { "ndcg@10": 0.12911785313248206, "recall@20": 0.06310061346836647, "mrr@10": 0.2541897059228082, "hitrate@10": 0.5320623916811091, "coverage@20": 0.15712, "novelty@20": 9.210487638047443, "ild@20": 0.7439924794488995, "tail_share@20": 0 }, "ml-32m": { "ndcg@10": 0.06358513640024253, "recall@20": 0.03921728966598933, "mrr@10": 0.13397612488521582, "hitrate@10": 0.34011443102352196, "coverage@20": 0.06930126002290951, "novelty@20": 9.876118825708627, "ild@20": 0.7255277488889702, "tail_share@20": 0 }, "retailrocket": { "ndcg@10": 0.004810860352094985, "recall@20": 0.024964080459770114, "mrr@10": 0.010344827586206896, "hitrate@10": 0.017241379310344827, "coverage@20": 0.22495361781076068, "novelty@20": 11.045246667116938, "ild@20": 1, "tail_share@20": 0.5310344827586206 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 1, "compared_on_n_datasets": 10 }, { "rank": 25, "key": "popularity", "system": "Popularity", "index": 38.7, "family": "non-personalised", "generative": false, "confidence": "reference", "origin": "Standard non-personalised baseline", "citation": "Cremonesi, Koren & Turrin, RecSys 2010 (TopPop).", "summary": "Ranks every slate by catalogue-wide interaction count, returning the same order for every user. It never reads who is asking, so nothing it produces is personalised. It is deterministic and has nothing to tune, which is why every release runs it as a fixed point of comparison; it sets no endpoint on the index scale and is scored like every other entrant.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 44.14, "cold_start": 45.26, "long_tail": 21.55, "beyond_accuracy": 25.34, "multi_objective": 33.84, "robustness": 48.78 }, "per_dataset_relevance": { "amazon-beauty-2023": 71.41304347826086, "goodreads-poetry": 24.347826086956523, "hm-fashion": 65.71428571428572, "kuairand-pure": 17.5, "kuairec": 60.92391304347825, "lastfm-1k": 70.47619047619047, "mind-small": 19.021739130434785, "ml-1m": 66.3586956521739, "ml-32m": 26.73913043478261, "retailrocket": 18.913043478260867 }, "n_datasets": 10, "spread": 23.129570197224297, "monthly_cost_usd": 2.0421664005752906, "train_cost_usd": 6.204790475850841e-7, "inference_cost_per_1k": 0.00020421637014914336, "latency_p50_ms": 0.11958798250606503, "latency_p95_ms": 0.11958798250606503, "model_mb": 0.1039488, "requires_gpu": false, "retrain_note": "counts only; recomputable in seconds, streamable", "latency_caveat": "", "value_index": 136.45950456970283, "other_track_index": 38.65, "win_rate": 35.97915797283098, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.01123438330702515, "recall@20": 0.021092524074696244, "mrr@10": 0.01781736111111111, "hitrate@10": 0.02675, "coverage@20": 0.001539645881447267, "novelty@20": 9.54572708925162, "ild@20": 1, "tail_share@20": 0 }, "goodreads-poetry": { "ndcg@10": 0.04437344845091332, "recall@20": 0.08440434713621157, "mrr@10": 0.058010912698412694, "hitrate@10": 0.11328125, "coverage@20": 0.01584567688598002, "novelty@20": 6.799385265167513, "ild@20": 0.4285957202576755, "tail_share@20": 0 }, "hm-fashion": { "ndcg@10": 0.007965302247933582, "recall@20": 0.013275361652790942, "mrr@10": 0.01578095238095238, "hitrate@10": 0.04375, "coverage@20": 0.0007253808249330899, "novelty@20": 10.458076787351667, "ild@20": 0.7552657146869153, "tail_share@20": 0 }, "kuairand-pure": { "ndcg@10": 0.009959300067741882, "recall@20": 0.020497662686032255, "mrr@10": 0.019263392857142857, "hitrate@10": 0.04875, "coverage@20": 0.005703674227351107, "novelty@20": 8.173523068921744, "ild@20": 0.42372500000000013, "tail_share@20": 0 }, "kuairec": { "ndcg@10": 0.056672023900436005, "recall@20": 0.06366305589295265, "mrr@10": 0.10379483952171177, "hitrate@10": 0.17709251101321585, "coverage@20": 0.043382756727073035, "novelty@20": 7.884705092618665, "ild@20": 0.7924715833488424, "tail_share@20": 0 }, "lastfm-1k": { "ndcg@10": 0.06212193825712727, "recall@20": 0.010983059339715589, "mrr@10": 0.1313284723662082, "hitrate@10": 0.28737300435413643, "coverage@20": 0.00766921525399878, "novelty@20": 10.379998778613656, "ild@20": 1, "tail_share@20": 0 }, "mind-small": { "ndcg@10": 0.048280806968302635, "recall@20": 0.16666178321270234, "mrr@10": 0.06438809523809524, "hitrate@10": 0.1295, "coverage@20": 0.008084074373484237, "novelty@20": 7.257732679186361, "ild@20": 0.8790256578947369, "tail_share@20": 0 }, "ml-1m": { "ndcg@10": 0.1780508548957317, "recall@20": 0.08113421152305585, "mrr@10": 0.3127369260818134, "hitrate@10": 0.5641247833622184, "coverage@20": 0.04736, "novelty@20": 8.257869704767545, "ild@20": 0.7245672026850509, "tail_share@20": 0 }, "ml-32m": { "ndcg@10": 0.06514838633521539, "recall@20": 0.036589134800343245, "mrr@10": 0.13685001866820048, "hitrate@10": 0.2956134774316593, "coverage@20": 0.014604810996563574, "novelty@20": 8.347723040299407, "ild@20": 0.724009345670191, "tail_share@20": 0 }, "retailrocket": { "ndcg@10": 0.0030925788817201625, "recall@20": 0.014906609195402298, "mrr@10": 0.0031130268199233715, "hitrate@10": 0.02586206896551724, "coverage@20": 0.012059369202226345, "novelty@20": 8.659850231005143, "ild@20": 1, "tail_share@20": 0 } }, "beats_next_significantly": true, "beats_next_on_n_datasets": 8, "compared_on_n_datasets": 10 }, { "rank": 26, "key": "content", "system": "Content TF-IDF", "index": 37.66, "family": "neighbourhood", "generative": false, "confidence": "reference", "origin": "Content-based filtering over item text and tags", "citation": "Standard content-based baseline; Lops et al., Recommender Systems Handbook 2011.", "summary": "TF-IDF over item titles and tags, scoring items by textual similarity to the items in a user's history. It reads no interaction data about the item being scored, so it can rank an item that has never been interacted with.", "stages": "single-stage", "proxy_of": null, "proxy_caveat": null, "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 14.79, "cold_start": 52.29, "long_tail": 83.79, "beyond_accuracy": 63.88, "multi_objective": 36.61, "robustness": 13.53 }, "per_dataset_relevance": { "amazon-beauty-2023": 7.934782608695652, "goodreads-poetry": 5, "hm-fashion": 35.357142857142854, "kuairand-pure": 4.3478260869565215, "kuairec": 73.80434782608695, "lastfm-1k": 8.690476190476192, "mind-small": 4.3478260869565215, "ml-1m": 2.8260869565217392, "ml-32m": 2.282608695652174, "retailrocket": 3.2608695652173907 }, "n_datasets": 10, "spread": 21.76306865611114, "monthly_cost_usd": 3.5904416407552304, "train_cost_usd": 0.0030245043958637896, "inference_cost_per_1k": 0.0003577285046633224, "latency_p50_ms": 0.2217888242927902, "latency_p95_ms": 0.2217888242927902, "model_mb": 43.9368212, "requires_gpu": false, "retrain_note": "TF-IDF refresh when the catalogue changes; no behavioural refit", "latency_caveat": "", "value_index": 75.52920063269343, "other_track_index": 35.36, "win_rate": 31.98347011638373, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.0005320624532478754, "recall@20": 0.0008343724457803101, "mrr@10": 0.0012165674603174601, "hitrate@10": 0.004, "coverage@20": 0.0033358994098024123, "novelty@20": 17.03649277690493, "ild@20": 1, "tail_share@20": 0.9011625 }, "goodreads-poetry": { "ndcg@10": 0.018743820567744634, "recall@20": 0.059559058020972076, "mrr@10": 0.021900059937169314, "hitrate@10": 0.07291666666666667, "coverage@20": 0.59800206682742, "novelty@20": 12.12465479150602, "ild@20": 0.2928836883947507, "tail_share@20": 0.38798828125 }, "hm-fashion": { "ndcg@10": 0.0035300141664737095, "recall@20": 0.008279128499248226, "mrr@10": 0.006928174603174603, "hitrate@10": 0.02175, "coverage@20": 0.4135671227394382, "novelty@20": 15.13369827327728, "ild@20": 0.3170553687290421, "tail_share@20": 0.2921125 }, "kuairand-pure": { "ndcg@10": 0.001306569213365825, "recall@20": 0.0029833161827342806, "mrr@10": 0.0029134920634920633, "hitrate@10": 0.00725, "coverage@20": 0.4334792412786842, "novelty@20": 15.045542208263408, "ild@20": 0.30377894736842115, "tail_share@20": 0.763175 }, "kuairec": { "ndcg@10": 0.053499316791794524, "recall@20": 0.0864820490783612, "mrr@10": 0.103415845045801, "hitrate@10": 0.2740088105726872, "coverage@20": 0.48599670510708404, "novelty@20": 12.637540853506733, "ild@20": 0.10283457735848395, "tail_share@20": 0.6449779735682819 }, "lastfm-1k": { "ndcg@10": 0.004348255033960462, "recall@20": 0.000333902114083399, "mrr@10": 0.013569239984334323, "hitrate@10": 0.033381712626995644, "coverage@20": 0.07286927154181716, "novelty@20": 16.946083441191146, "ild@20": 1, "tail_share@20": 0.5282293178519594 }, "mind-small": { "ndcg@10": 0.00444615289904792, "recall@20": 0.0149934613997114, "mrr@10": 0.005361011904761905, "hitrate@10": 0.01775, "coverage@20": 0.9490703314470493, "novelty@20": 13.240815161824678, "ild@20": 0.1016375000000001, "tail_share@20": 0.495875 }, "ml-1m": { "ndcg@10": 0.025895379210304905, "recall@20": 0.015162106699902323, "mrr@10": 0.06513954224092872, "hitrate@10": 0.18110918544194107, "coverage@20": 0.30848, "novelty@20": 12.845769651760456, "ild@20": 0.3568432894095318, "tail_share@20": 0.4540727902946274 }, "ml-32m": { "ndcg@10": 0.011742982650376753, "recall@20": 0.0069407184183545405, "mrr@10": 0.03295820341274886, "hitrate@10": 0.09090909090909091, "coverage@20": 0.10385643375334097, "novelty@20": 15.214790861750744, "ild@20": 0.32980115106415586, "tail_share@20": 0.39637635092180545 }, "retailrocket": { "ndcg@10": 0, "recall@20": 0, "mrr@10": 0, "hitrate@10": 0, "coverage@20": 0.03153988868274583, "novelty@20": 11.632382930323418, "ild@20": 1, "tail_share@20": 0.759051724137931 } }, "beats_next_significantly": false, "beats_next_on_n_datasets": 2, "compared_on_n_datasets": 10 }, { "rank": 27, "key": "phoenix_style", "system": "Phoenix-style unified transformer", "index": 27.09, "family": "unified-transformer", "generative": false, "confidence": "proxy", "origin": "xAI / X — Phoenix", "citation": "xAI, 'x-algorithm' (github.com/xai-org/x-algorithm), phoenix/README.md, January 2026.", "summary": "One transformer family performing both retrieval and ranking, with items represented by residual-quantised semantic codes and hashed author ids and no hand- engineered features. A ranking attention mask stops candidates attending to one another, so an item's score does not depend on which other items it was scored alongside; the harness verifies that property directly.", "stages": "two-tower retrieval + candidate-isolated ranking", "proxy_of": "Phoenix (X / xAI)", "proxy_caveat": "Phoenix in production is a Grok-derived transformer with a 2560-dimensional embedding, eight layers, 1022 history positions and a 64-action taxonomy, trained on X's own logs with a multimodal post encoder. This entrant implements the published architecture at benchmark scale, with the harness's own semantic-ID tokenizer and whatever action taxonomy the dataset actually logs — often a single one. It is not X's model.", "introduced_in": null, "native_retriever": null, "retriever_label": null, "ranker_label": null, "is_new": false, "dimensions": { "relevance": 13.33, "cold_start": 29.82, "long_tail": 44.62, "beyond_accuracy": 57.91, "multi_objective": 30.54, "robustness": 16.09 }, "per_dataset_relevance": { "amazon-beauty-2023": 12.391304347826086, "goodreads-poetry": 6.8478260869565215, "hm-fashion": 11.547619047619047, "kuairand-pure": 18.47826086956522, "kuairec": 7.17391304347826, "lastfm-1k": 30.35714285714286, "mind-small": 11.195652173913045, "ml-1m": 12.82608695652174, "ml-32m": 10.326086956521738, "retailrocket": 12.17391304347826 }, "n_datasets": 10, "spread": 6.446484068229971, "monthly_cost_usd": 5.616773804874496, "train_cost_usd": 0.15255801485757525, "inference_cost_per_1k": 0.0004953146440244044, "latency_p50_ms": 0.2924021924309524, "latency_p95_ms": 0.2924021924309524, "model_mb": 19.738995199999998, "requires_gpu": true, "retrain_note": "tokenizer refit when the catalogue shifts, then a model retrain; the tokenizer is the harder half because changing item codes invalidates the model", "latency_caveat": "", "value_index": 34.727189746417665, "other_track_index": 32.69, "win_rate": 20.449645291418985, "raw": { "amazon-beauty-2023": { "ndcg@10": 0.0010819934417871072, "recall@20": 0.0034785561660561656, "mrr@10": 0.0010703373015873018, "hitrate@10": 0.00475, "coverage@20": 0.011162432640492687, "novelty@20": 13.618577707614245, "ild@20": 1, "tail_share@20": 0.2515625 }, "goodreads-poetry": { "ndcg@10": 0.029910598763080252, "recall@20": 0.057890450632190116, "mrr@10": 0.041069361772486775, "hitrate@10": 0.103515625, "coverage@20": 0.0775060282466414, "novelty@20": 9.043951734264075, "ild@20": 0.4469439590186404, "tail_share@20": 0.010677083333333334 }, "hm-fashion": { "ndcg@10": 0.0007047316591421382, "recall@20": 0.0010161876608644341, "mrr@10": 0.0019357142857142854, "hitrate@10": 0.00475, "coverage@20": 0.10715625703494334, "novelty@20": 14.615723046579282, "ild@20": 0.43285861408360293, "tail_share@20": 0.21595 }, "kuairand-pure": { "ndcg@10": 0.010117304679510008, "recall@20": 0.023405126879562716, "mrr@10": 0.016208829365079367, "hitrate@10": 0.0435, "coverage@20": 0.03223239156386789, "novelty@20": 9.558122097074703, "ild@20": 0.40522039473684224, "tail_share@20": 0 }, "kuairec": { "ndcg@10": 0.014513445760685743, "recall@20": 0.020485506041940913, "mrr@10": 0.03389553178099433, "hitrate@10": 0.1013215859030837, "coverage@20": 0.0686436024162548, "novelty@20": 9.60566075995481, "ild@20": 0.811550875339144, "tail_share@20": 0.001277533039647577 }, "lastfm-1k": { "ndcg@10": 0.023601024162656158, "recall@20": 0.0016186139012996973, "mrr@10": 0.06361416361416361, "hitrate@10": 0.14949201741654572, "coverage@20": 0.038369529527651394, "novelty@20": 12.843163071916514, "ild@20": 1, "tail_share@20": 0.048040638606676345 }, "mind-small": { "ndcg@10": 0.03597409228044063, "recall@20": 0.05441469950006134, "mrr@10": 0.05291845238095238, "hitrate@10": 0.09975, "coverage@20": 0.026407976286715172, "novelty@20": 9.764632079960505, "ild@20": 0.9473447368421053, "tail_share@20": 0.0749 }, "ml-1m": { "ndcg@10": 0.11543462968191048, "recall@20": 0.04638756848144572, "mrr@10": 0.2208206101620313, "hitrate@10": 0.46533795493934144, "coverage@20": 0.10976, "novelty@20": 9.428095046520323, "ild@20": 0.6721554638346744, "tail_share@20": 0.006759098786828423 }, "ml-32m": { "ndcg@10": 0.058652825930861914, "recall@20": 0.028640244738894653, "mrr@10": 0.13243221425039609, "hitrate@10": 0.2828989192625556, "coverage@20": 0.014700267277586864, "novelty@20": 9.573242467756332, "ild@20": 0.7228196951551045, "tail_share@20": 0.0008264462809917355 }, "retailrocket": { "ndcg@10": 0.003018519844685174, "recall@20": 0.008620689655172414, "mrr@10": 0.003232758620689655, "hitrate@10": 0.017241379310344827, "coverage@20": 0.22356215213358072, "novelty@20": 10.833210430251668, "ild@20": 1, "tail_share@20": 0.39870689655172414 } }, "beats_next_significantly": null, "beats_next_on_n_datasets": null, "compared_on_n_datasets": null } ], "reference_marks": { "floor_system": "popularity", "reference_system": "itemknn", "floor_score": 38.7, "floor_rank": 25, "reference_score": 58.13, "reference_rank": 9, "n_ranked": 27, "note": "Two deterministic, hyperparameter-free systems that every release is required to run. They set no endpoint on the scale and are measured like every other entrant — their scores are shown so a reader who knows what these two are can read the rest of the board off them." }, "secondary": { "best_overall": { "winner": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 68.81834239130434, "criterion": "Recommender Performance Index", "ranking": [ { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 68.81834239130434 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 66.88849545252883 }, { "system": "vsknn", "label": "VS-KNN", "value": 65.16611856699201 }, { "system": "ease", "label": "EASEᴿ", "value": 64.51358695652173 }, { "system": "sasrec", "label": "SASRec", "value": 62.8668191733215 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 60.75593605195701 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 60.16425706152027 }, { "system": "lightgcn", "label": "LightGCN", "value": 59.840297187715656 }, { "system": "itemknn", "label": "ItemKNN", "value": 58.12506315932169 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 57.18902537464262 } ] }, "best_generative": { "winner": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 55.04977293453612, "criterion": "Recommender Performance Index, generative entrants only", "ranking": [ { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 55.04977293453612 }, { "system": "tiger", "label": "TIGER", "value": 51.60373379424232 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 49.64568267524401 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 45.97227613871636 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 42.365217083210084 } ] }, "best_cold_start": { "winner": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 63.92921842650104, "criterion": "cold-start dimension", "ranking": [ { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 63.92921842650104 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 61.22203120378587 }, { "system": "vsknn", "label": "VS-KNN", "value": 60.36444469091985 }, { "system": "sasrec", "label": "SASRec", "value": 59.65060263235729 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 59.326438183969245 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 58.84048728186926 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 58.07745489500148 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 57.676778320023665 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 56.675613723750374 }, { "system": "ease", "label": "EASEᴿ", "value": 54.54710144927535 } ] }, "best_long_tail": { "winner": "content", "label": "Content TF-IDF", "value": 83.7883022774327, "criterion": "long-tail dimension", "ranking": [ { "system": "content", "label": "Content TF-IDF", "value": 83.7883022774327 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 78.30693581780538 }, { "system": "ials", "label": "iALS", "value": 67.8659420289855 }, { "system": "sasrec", "label": "SASRec", "value": 67.53364389233955 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 66.40734989648033 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 66.09472049689441 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 65.45703933747411 }, { "system": "lightgcn", "label": "LightGCN", "value": 60.39699792960662 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 59.95134575569358 }, { "system": "tiger", "label": "TIGER", "value": 59.12836438923395 } ] }, "best_multi_objective": { "winner": "sasrec", "label": "SASRec", "value": 81.41304347826087, "criterion": "multi-objective dimension", "ranking": [ { "system": "sasrec", "label": "SASRec", "value": 81.41304347826087 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 70.57971014492753 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 69.63768115942028 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 68.22463768115941 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 67.7536231884058 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 65.39855072463769 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 65.16304347826087 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 63.75 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 62.80797101449276 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 61.630434782608695 } ] }, "best_beyond_accuracy": { "winner": "gru4rec", "label": "GRU4Rec", "value": 72.6182712215321, "criterion": "novelty, diversity and coverage", "ranking": [ { "system": "gru4rec", "label": "GRU4Rec", "value": 72.6182712215321 }, { "system": "ials", "label": "iALS", "value": 70.66045548654245 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 69.8682712215321 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 66.61309523809524 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 66.32472826086956 }, { "system": "content", "label": "Content TF-IDF", "value": 63.880693581780534 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 63.75957556935817 }, { "system": "sasrec", "label": "SASRec", "value": 61.13121118012422 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 58.83876811594203 }, { "system": "lightgcn", "label": "LightGCN", "value": 58.14104554865424 } ] }, "best_quality_per_dollar": { "winner": "recency_pop", "label": "Recency-Weighted Popularity", "value": 298.6294735374231, "criterion": "Recommender Value Index", "ranking": [ { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 298.6294735374231 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 264.7163761566911 }, { "system": "sasrec", "label": "SASRec", "value": 247.732038975163 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 240.18629832127954 }, { "system": "lightgcn", "label": "LightGCN", "value": 229.59472889414624 }, { "system": "popularity", "label": "Popularity", "value": 136.45950456970283 }, { "system": "ials", "label": "iALS", "value": 128.4353220999625 }, { "system": "itemknn", "label": "ItemKNN", "value": 100 }, { "system": "ease", "label": "EASEᴿ", "value": 97.59528105289725 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 96.12082882547067 } ] }, "best_under_latency": { "winner": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 68.81834239130434, "criterion": "Recommender Performance Index among systems serving under 25 ms/user offline", "ranking": [ { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 68.81834239130434 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 66.88849545252883 }, { "system": "vsknn", "label": "VS-KNN", "value": 65.16611856699201 }, { "system": "ease", "label": "EASEᴿ", "value": 64.51358695652173 }, { "system": "sasrec", "label": "SASRec", "value": 62.8668191733215 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 60.75593605195701 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 60.16425706152027 }, { "system": "lightgcn", "label": "LightGCN", "value": 59.840297187715656 }, { "system": "itemknn", "label": "ItemKNN", "value": 58.12506315932169 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 57.18902537464262 } ] }, "best_retriever": { "winner": "ease", "label": "EASEᴿ", "value": 0.20821024393646054, "criterion": "candidate recall@100, systems with a separable retrieval stage", "ranking": [ { "system": "ease", "label": "EASEᴿ", "value": 0.20821024393646054 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.20821024393646054 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.19384892477679774 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.1817850328838331 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.1817850328838331 }, { "system": "sasrec", "label": "SASRec", "value": 0.178106046221883 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.17372499806588007 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.16411816511728064 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.1637260884908064 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.16016210561913946 } ] }, "best_ranker": { "winner": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.07005531610320892, "criterion": "NDCG@10 on identical candidate sets", "ranking": [ { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.07005531610320892 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.0694144505795006 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.06824009229273888 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.06806640850716633 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.06688574222914205 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.06322828062908421 }, { "system": "sasrec", "label": "SASRec", "value": 0.060300068536714026 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.060290959271488906 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.058256059288303484 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.055420470400156165 } ] } }, "pairwise_provenance": { "metric": "ndcg@10", "alpha": 0.05, "n_permutations": 2000, "n_pairs_tested_this_release": 150, "cells_from_stored_vectors": 30, "cells_carried_from_within_run_grids": 504, "datasets_with_no_stored_vectors": { "amazon-beauty-2023": [ "v3_best_beauty" ], "goodreads-poetry": [ "v3_best_goodreads" ], "hm-fashion": [ "v3_best_hm" ], "kuairand-pure": [ "v3_best_kuairand" ], "kuairec": [ "v3_best_kuairec" ], "lastfm-1k": [ "v3_best_lastfm" ], "mind-small": [ "v3_best_mind" ], "ml-1m": [ "v3_best_ml1m" ], "ml-32m": [ "v3_best_ml32m" ], "retailrocket": [ "v3_best_retailrocket" ] }, "unpairable": [], "note": "Pairs are tested from per-user vectors stored in the result files, so two systems need not have run in the same job. Cells marked as carried forward come from a release written before those vectors were stored; they are that release's own tests, not new ones." }, "scenarios": { "cold_users": { "description": "Users with 1–4 prior interactions", "metric": "cohort:cold_users_1_4:ndcg@10", "datasets": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "mind-small", "ml-1m", "retailrocket" ], "n_datasets": 7, "ranking": [ { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.09825406409501883, "n_datasets": 7 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.08638339232559966, "n_datasets": 7 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.08470354399342149, "n_datasets": 7 }, { "system": "ease", "label": "EASEᴿ", "value": 0.08383515915197769, "n_datasets": 6 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.08144105222768831, "n_datasets": 7 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.08002963758619956, "n_datasets": 7 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.07960913592526887, "n_datasets": 7 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.07929782564129381, "n_datasets": 7 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.07561223294378146, "n_datasets": 6 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.07394280819711144, "n_datasets": 7 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.07387614040953236, "n_datasets": 7 }, { "system": "sasrec", "label": "SASRec", "value": 0.07214872337393217, "n_datasets": 7 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.07100458522225102, "n_datasets": 7 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.07091606075133619, "n_datasets": 7 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.07052744903152915, "n_datasets": 7 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.0702941794200824, "n_datasets": 7 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.0695615500312126, "n_datasets": 7 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.06946922688106064, "n_datasets": 7 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.06899911600930415, "n_datasets": 7 }, { "system": "tiger", "label": "TIGER", "value": 0.06890773202697612, "n_datasets": 7 }, { "system": "popularity", "label": "Popularity", "value": 0.06281402100054413, "n_datasets": 7 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.0625249088797488, "n_datasets": 7 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.05328658841240781, "n_datasets": 7 }, { "system": "ials", "label": "iALS", "value": 0.05187699161805665, "n_datasets": 7 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.04644933614084589, "n_datasets": 7 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.03916363060665102, "n_datasets": 7 }, { "system": "content", "label": "Content TF-IDF", "value": 0.011946801946541158, "n_datasets": 7 } ], "not_measured": [] }, "cold_items": { "description": "Target items with no fit-window interactions", "metric": "cohort:cold_items:ndcg@10", "datasets": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairec" ], "n_datasets": 4, "ranking": [ { "system": "content", "label": "Content TF-IDF", "value": 0.016578138495828388, "n_datasets": 4 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.005487515956883971, "n_datasets": 4 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.003896738291346104, "n_datasets": 4 }, { "system": "popularity", "label": "Popularity", "value": 0, "n_datasets": 4 }, { "system": "itemknn", "label": "ItemKNN", "value": 0, "n_datasets": 4 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0, "n_datasets": 4 }, { "system": "ease", "label": "EASEᴿ", "value": 0, "n_datasets": 3 }, { "system": "ials", "label": "iALS", "value": 0, "n_datasets": 4 }, { "system": "vsknn", "label": "VS-KNN", "value": 0, "n_datasets": 4 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0, "n_datasets": 4 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0, "n_datasets": 4 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0, "n_datasets": 3 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0, "n_datasets": 4 }, { "system": "lightgcn", "label": "LightGCN", "value": 0, "n_datasets": 4 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0, "n_datasets": 4 }, { "system": "sasrec", "label": "SASRec", "value": 0, "n_datasets": 4 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0, "n_datasets": 4 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0, "n_datasets": 4 }, { "system": "tiger", "label": "TIGER", "value": 0, "n_datasets": 4 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0, "n_datasets": 4 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0, "n_datasets": 4 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0, "n_datasets": 4 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0, "n_datasets": 4 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0, "n_datasets": 4 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0, "n_datasets": 4 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0, "n_datasets": 4 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0, "n_datasets": 4 } ], "not_measured": [] }, "sparse_histories": { "description": "Bottom-quartile history length", "metric": "cohort:sparse_history:ndcg@10", "datasets": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "n_datasets": 10, "ranking": [ { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.09469966075883292, "n_datasets": 10 }, { "system": "ease", "label": "EASEᴿ", "value": 0.0911158497894255, "n_datasets": 8 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.08730137867409152, "n_datasets": 10 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.08554762231256265, "n_datasets": 10 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.08443506149460078, "n_datasets": 10 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.08393106311141568, "n_datasets": 10 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.08341970295229136, "n_datasets": 10 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.07194391008013816, "n_datasets": 10 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.07113608342321477, "n_datasets": 8 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.06631153789702682, "n_datasets": 10 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.06317736023408192, "n_datasets": 10 }, { "system": "popularity", "label": "Popularity", "value": 0.06304506134804803, "n_datasets": 10 }, { "system": "sasrec", "label": "SASRec", "value": 0.06199961344917314, "n_datasets": 10 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.061960707943955994, "n_datasets": 10 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.06187593218857512, "n_datasets": 10 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.06084769196624963, "n_datasets": 10 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.06024412159482888, "n_datasets": 10 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.0600806617911683, "n_datasets": 10 }, { "system": "ials", "label": "iALS", "value": 0.0589971647984752, "n_datasets": 10 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.0575334063847299, "n_datasets": 10 }, { "system": "tiger", "label": "TIGER", "value": 0.057346819106247046, "n_datasets": 10 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.05383151286747462, "n_datasets": 10 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.05286056746605118, "n_datasets": 10 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.05157566988233444, "n_datasets": 10 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.04178350276056679, "n_datasets": 10 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.03632416837575221, "n_datasets": 10 }, { "system": "content", "label": "Content TF-IDF", "value": 0.011036098899569888, "n_datasets": 10 } ], "not_measured": [] }, "dense_histories": { "description": "Top-quartile history length", "metric": "cohort:dense_history:ndcg@10", "datasets": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m" ], "n_datasets": 9, "ranking": [ { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.07561093088265233, "n_datasets": 9 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.05910602071753347, "n_datasets": 9 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.05867217998428777, "n_datasets": 9 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.058453463918236644, "n_datasets": 9 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.057612565469466887, "n_datasets": 9 }, { "system": "ease", "label": "EASEᴿ", "value": 0.056629044385787454, "n_datasets": 7 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.05656256242223996, "n_datasets": 7 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.05499446814248637, "n_datasets": 9 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.05283804311934428, "n_datasets": 9 }, { "system": "sasrec", "label": "SASRec", "value": 0.05243363803279429, "n_datasets": 9 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.05188172108978473, "n_datasets": 9 }, { "system": "tiger", "label": "TIGER", "value": 0.051500986790054944, "n_datasets": 9 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.050168445503752275, "n_datasets": 9 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.048925133437394734, "n_datasets": 9 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.048232678834394, "n_datasets": 9 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.04745590258013215, "n_datasets": 9 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.046595708239659514, "n_datasets": 9 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.04608337614741847, "n_datasets": 9 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.04518535238187811, "n_datasets": 9 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.04298780892601304, "n_datasets": 9 }, { "system": "popularity", "label": "Popularity", "value": 0.0428631035304519, "n_datasets": 9 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.04249134195624818, "n_datasets": 9 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.03984638796338883, "n_datasets": 9 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.035124567270542105, "n_datasets": 9 }, { "system": "ials", "label": "iALS", "value": 0.03066518440331752, "n_datasets": 9 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.027973366264373443, "n_datasets": 9 }, { "system": "content", "label": "Content TF-IDF", "value": 0.014915110319010806, "n_datasets": 9 } ], "not_measured": [] }, "long_tail": { "description": "Targets below median popularity", "metric": "cohort:tail_target_items:ndcg@10", "datasets": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "mind-small", "ml-32m", "retailrocket" ], "n_datasets": 8, "ranking": [ { "system": "content", "label": "Content TF-IDF", "value": 0.01031157315209764, "n_datasets": 8 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.0038891046264599716, "n_datasets": 8 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.0035650533827470093, "n_datasets": 8 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.003517675666992249, "n_datasets": 8 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.0032849294945116584, "n_datasets": 7 }, { "system": "ease", "label": "EASEᴿ", "value": 0.002682050655685108, "n_datasets": 7 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.0026526880302117266, "n_datasets": 8 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.0025024601849865137, "n_datasets": 8 }, { "system": "ials", "label": "iALS", "value": 0.0024265956626704148, "n_datasets": 8 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.002081305979538917, "n_datasets": 8 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.0018861274009295682, "n_datasets": 8 }, { "system": "tiger", "label": "TIGER", "value": 0.0017253436057820593, "n_datasets": 8 }, { "system": "sasrec", "label": "SASRec", "value": 0.0016994655375108373, "n_datasets": 8 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.0013352712777311118, "n_datasets": 8 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.001265096408822896, "n_datasets": 8 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.0010305045223412803, "n_datasets": 8 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.0007901699277994546, "n_datasets": 8 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.0006516404346777793, "n_datasets": 8 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.0005382256056399785, "n_datasets": 8 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.0005217658198093584, "n_datasets": 8 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.0005203152143243679, "n_datasets": 8 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.0003482959328674975, "n_datasets": 8 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.0003285156354749248, "n_datasets": 8 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.00032140117872599323, "n_datasets": 8 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.000038331800061330875, "n_datasets": 8 }, { "system": "popularity", "label": "Popularity", "value": 0, "n_datasets": 8 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0, "n_datasets": 8 } ], "not_measured": [] }, "catalogue_coverage": { "description": "Share of the catalogue ever served", "metric": "coverage@20", "datasets": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "n_datasets": 10, "ranking": [ { "system": "gru4rec", "label": "GRU4Rec", "value": 0.511532054867162, "n_datasets": 10 }, { "system": "ials", "label": "iALS", "value": 0.4408942768430194, "n_datasets": 10 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.38386932502644594, "n_datasets": 10 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.34506315600587945, "n_datasets": 10 }, { "system": "content", "label": "Content TF-IDF", "value": 0.3400196960787383, "n_datasets": 10 }, { "system": "ease", "label": "EASEᴿ", "value": 0.3279957479993004, "n_datasets": 8 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.3244323583486136, "n_datasets": 10 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.3157148076796474, "n_datasets": 10 }, { "system": "sasrec", "label": "SASRec", "value": 0.308017056769827, "n_datasets": 10 }, { "system": "tiger", "label": "TIGER", "value": 0.28514613637657427, "n_datasets": 10 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.28487697388466726, "n_datasets": 8 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.27924216977584687, "n_datasets": 10 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.25852039815786637, "n_datasets": 10 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.257451186908066, "n_datasets": 10 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.24932758379246875, "n_datasets": 10 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.21846407730861395, "n_datasets": 10 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.2164809516581429, "n_datasets": 10 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.21131557428683392, "n_datasets": 10 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.196348349351349, "n_datasets": 10 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.18386507899408527, "n_datasets": 10 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.1715575334796761, "n_datasets": 10 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.17023425391469688, "n_datasets": 10 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.128964396300903, "n_datasets": 10 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.12421080467884209, "n_datasets": 10 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.07095006371277343, "n_datasets": 10 }, { "system": "popularity", "label": "Popularity", "value": 0.015697460437305747, "n_datasets": 10 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.014271662098529456, "n_datasets": 10 } ], "not_measured": [] }, "novelty": { "description": "Mean self-information of served items", "metric": "novelty@20", "datasets": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "n_datasets": 10, "ranking": [ { "system": "content", "label": "Content TF-IDF", "value": 14.18577709503088, "n_datasets": 10 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 11.401893737518332, "n_datasets": 10 }, { "system": "sasrec", "label": "SASRec", "value": 11.1379439469363, "n_datasets": 10 }, { "system": "ials", "label": "iALS", "value": 11.013923031975441, "n_datasets": 10 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 10.991153076934712, "n_datasets": 10 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 10.89479615472882, "n_datasets": 10 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 10.888437844189244, "n_datasets": 10 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 10.85277129012196, "n_datasets": 10 }, { "system": "tiger", "label": "TIGER", "value": 10.714047439620368, "n_datasets": 10 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 10.57155765506719, "n_datasets": 10 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 10.558922608401685, "n_datasets": 10 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 10.319468911606453, "n_datasets": 10 }, { "system": "lightgcn", "label": "LightGCN", "value": 10.316370433425373, "n_datasets": 10 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 10.291980693814295, "n_datasets": 10 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 10.247654008003343, "n_datasets": 10 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 10.157360445768209, "n_datasets": 10 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 10.089703918747663, "n_datasets": 10 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 10.0517524060961, "n_datasets": 8 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 10.015746141789014, "n_datasets": 10 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 9.994370757499283, "n_datasets": 10 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 9.993673367895237, "n_datasets": 10 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 9.977701456823448, "n_datasets": 10 }, { "system": "itemknn", "label": "ItemKNN", "value": 9.597458051598148, "n_datasets": 10 }, { "system": "ease", "label": "EASEᴿ", "value": 9.564352764510168, "n_datasets": 8 }, { "system": "vsknn", "label": "VS-KNN", "value": 9.45929269903302, "n_datasets": 10 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 9.244666845248842, "n_datasets": 10 }, { "system": "popularity", "label": "Popularity", "value": 8.57645917371833, "n_datasets": 10 } ], "not_measured": [] }, "diversity": { "description": "Intra-list tag diversity", "metric": "ild@20", "datasets": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "n_datasets": 10, "ranking": [ { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.794070453594628, "n_datasets": 10 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.791031905560881, "n_datasets": 10 }, { "system": "ials", "label": "iALS", "value": 0.7759908136942163, "n_datasets": 10 }, { "system": "popularity", "label": "Popularity", "value": 0.7727660224543412, "n_datasets": 10 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.7694561132397372, "n_datasets": 10 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.7670525665878618, "n_datasets": 10 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.7650801571202693, "n_datasets": 10 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.7605303772799826, "n_datasets": 10 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.7604655496208237, "n_datasets": 10 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.7596980274523187, "n_datasets": 10 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.7570248711741041, "n_datasets": 10 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.7555209099742894, "n_datasets": 8 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.7544160222330609, "n_datasets": 10 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.7540158409161302, "n_datasets": 10 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.7483950760089205, "n_datasets": 10 }, { "system": "ease", "label": "EASEᴿ", "value": 0.7460756070092893, "n_datasets": 8 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.7438893739010114, "n_datasets": 10 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.7354667763928291, "n_datasets": 10 }, { "system": "sasrec", "label": "SASRec", "value": 0.7323954794051345, "n_datasets": 10 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.7195442794915896, "n_datasets": 10 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.7190828058429519, "n_datasets": 10 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.7169084501790717, "n_datasets": 10 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.7162218853598652, "n_datasets": 10 }, { "system": "tiger", "label": "TIGER", "value": 0.7149781490605351, "n_datasets": 10 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.7123357335064116, "n_datasets": 10 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.7104693696167657, "n_datasets": 10 }, { "system": "content", "label": "Content TF-IDF", "value": 0.4804834522324386, "n_datasets": 10 } ], "not_measured": [] }, "multi_objective": { "description": "Weighted value across all logged signals", "metric": "mo_weighted_utility", "datasets": [ "kuairand-pure", "kuairec", "mind-small" ], "n_datasets": 3, "ranking": [ { "system": "sasrec", "label": "SASRec", "value": 0.05816451985957866, "n_datasets": 3 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.05596262449106206, "n_datasets": 3 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.049707005541190306, "n_datasets": 3 }, { "system": "content", "label": "Content TF-IDF", "value": 0.0472349771865991, "n_datasets": 3 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.04479601760434842, "n_datasets": 3 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.04428467131861211, "n_datasets": 3 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.04375392920570015, "n_datasets": 3 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.04348652124545055, "n_datasets": 3 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.0434655907706879, "n_datasets": 3 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.042100193945946246, "n_datasets": 3 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.04158835245802347, "n_datasets": 3 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.04130447896541204, "n_datasets": 3 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.04107925908824474, "n_datasets": 3 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.04091430372429944, "n_datasets": 3 }, { "system": "tiger", "label": "TIGER", "value": 0.040897503909189815, "n_datasets": 3 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.039411883854691375, "n_datasets": 3 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.037289393762119545, "n_datasets": 3 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.03615862566876116, "n_datasets": 3 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.030339907540077527, "n_datasets": 3 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.030024517682816134, "n_datasets": 3 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.02800087447883724, "n_datasets": 3 }, { "system": "ease", "label": "EASEᴿ", "value": 0.026873366305239798, "n_datasets": 3 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.025037504945625216, "n_datasets": 3 }, { "system": "popularity", "label": "Popularity", "value": 0.023098789568255684, "n_datasets": 3 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.022576931754184668, "n_datasets": 3 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.020836462472117862, "n_datasets": 3 }, { "system": "ials", "label": "iALS", "value": 0.01556104462326731, "n_datasets": 3 } ], "not_measured": [] } }, "generative_vs_traditional": { "generative": { "n": 5, "mean": 48.928000000000004, "best": 55.05, "median": 49.65 }, "traditional": { "n": 22, "mean": 53.49045454545455, "best": 68.82, "median": 52.875 }, "by_scenario": { "cold_users": { "generative_mean": 0.06181292288074815, "traditional_mean": 0.07035505815661966, "generative_best": 0.06946922688106064, "traditional_best": 0.09825406409501883, "winner": "traditional" }, "cold_items": { "generative_mean": 0, "traditional_mean": 0.0011801087610935665, "generative_best": 0, "traditional_best": 0.016578138495828388, "winner": "traditional" }, "sparse_histories": { "generative_mean": 0.0530770393165745, "traditional_mean": 0.06677166977127318, "generative_best": 0.06084769196624963, "traditional_best": 0.09469966075883292, "winner": "traditional" }, "dense_histories": { "generative_mean": 0.045353177544692466, "traditional_mean": 0.04859287295405372, "generative_best": 0.05283804311934428, "traditional_best": 0.07561093088265233, "winner": "traditional" }, "long_tail": { "generative_mean": 0.0012890595087031564, "traditional_mean": 0.0019758458906765394, "generative_best": 0.0018861274009295682, "traditional_best": 0.01031157315209764, "winner": "traditional" }, "catalogue_coverage": { "generative_mean": 0.30991972563731235, "traditional_mean": 0.2311302048431506, "generative_best": 0.34506315600587945, "traditional_best": 0.511532054867162, "winner": "traditional" }, "novelty": { "generative_mean": 10.689427743781028, "traditional_mean": 10.348277147145305, "generative_best": 10.991153076934712, "traditional_best": 14.18577709503088, "winner": "traditional" }, "diversity": { "generative_mean": 0.7427541224671195, "traditional_mean": 0.7372543389351754, "generative_best": 0.7650801571202693, "traditional_best": 0.794070453594628, "winner": "traditional" }, "multi_objective": { "generative_mean": 0.03881688782615221, "traditional_mean": 0.03726658240179898, "generative_best": 0.04428467131861211, "traditional_best": 0.05816451985957866, "winner": "traditional" } }, "generative_systems": [ "plum_style", "tiger", "hstu", "netflix_foundation_style", "onerec_style" ], "caveat": "three of the generative entrants are architecture proxies for proprietary systems and one is trained from scratch where the original is adapted from a pre-trained language model; the comparison is between architectures at benchmark scale, not between the companies' production systems" }, "per_dataset_winners": { "amazon-beauty-2023": { "winner": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.012392088447482793, "runner_up": "itemknn_mlp", "runner_up_value": 0.012028371092285375, "relative_gap": 0.030238288493667685, "ranking": [ { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.012392088447482793 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.012028371092285375 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.011660521941527292 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.011625749482506763 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.01151165901283288 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.011302085476693457 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.011234498400048335 }, { "system": "popularity", "label": "Popularity", "value": 0.01123438330702515 }, { "system": "tiger", "label": "TIGER", "value": 0.009996357604614182 }, { "system": "ease", "label": "EASEᴿ", "value": 0.009909636264594113 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.008631922985996486 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.00803095081958099 }, { "system": "sasrec", "label": "SASRec", "value": 0.008009979999979405 }, { "system": "ials", "label": "iALS", "value": 0.007548808509471626 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.007243294091116154 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.007219419201085 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.006627882916288606 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.006627570245643457 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.006586112144121495 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.0043700062684608925 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.0032981664432874245 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.00317932858246893 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.002901487000184543 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.0028046051157846834 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.0016514739896686193 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.0010819934417871072 }, { "system": "content", "label": "Content TF-IDF", "value": 0.0005320624532478754 } ] }, "goodreads-poetry": { "winner": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.12236973644812514, "runner_up": "lightgcn", "runner_up_value": 0.08294957604906816, "relative_gap": 0.4752304023318732, "ranking": [ { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.12236973644812514 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.08294957604906816 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.07603617612829523 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.0741359912242087 }, { "system": "ease", "label": "EASEᴿ", "value": 0.07349049382195426 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.07014321820636503 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.06823062024236214 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.06804370142959602 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.06783914059767854 }, { "system": "sasrec", "label": "SASRec", "value": 0.06699482402456493 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.06654433780230559 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.06609502835153398 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.06549798991315328 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.06308252019023752 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.06301065378539816 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.06171544728927122 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.06006612101389561 }, { "system": "ials", "label": "iALS", "value": 0.05686437652928566 }, { "system": "tiger", "label": "TIGER", "value": 0.055181491534953164 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.054967496844214786 }, { "system": "popularity", "label": "Popularity", "value": 0.04437344845091332 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.04271781966457821 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.03926248061987808 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.032304815626998294 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.029910598763080252 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.028119216305702938 }, { "system": "content", "label": "Content TF-IDF", "value": 0.018743820567744634 } ] }, "hm-fashion": { "winner": "vsknn", "label": "VS-KNN", "value": 0.011409717119931266, "runner_up": "itemknn", "runner_up_value": 0.009544831845529962, "relative_gap": 0.19538167927753147, "ranking": [ { "system": "vsknn", "label": "VS-KNN", "value": 0.011409717119931266 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.009544831845529962 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.008780587999060112 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.008739844553529082 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.008458184185904126 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.008108545596593946 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.00806584568245662 }, { "system": "popularity", "label": "Popularity", "value": 0.007965302247933582 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.007965302247933582 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.0074171507171370285 }, { "system": "ials", "label": "iALS", "value": 0.006604990358683441 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.006087679830425097 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.005719845682127772 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.005015206696320805 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.004582764196301507 }, { "system": "tiger", "label": "TIGER", "value": 0.004517793423918171 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.004487840874978534 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.004012639956827644 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.0038824928080265593 }, { "system": "content", "label": "Content TF-IDF", "value": 0.0035300141664737095 }, { "system": "sasrec", "label": "SASRec", "value": 0.0035110235937032865 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.0034191030199448038 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.003169243109474735 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.0021237139984091255 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.0007047316591421382 } ] }, "kuairand-pure": { "winner": "sasrec", "label": "SASRec", "value": 0.019280831768444688, "runner_up": "option_cascade", "runner_up_value": 0.018171526532123665, "relative_gap": 0.061046342714245326, "ranking": [ { "system": "sasrec", "label": "SASRec", "value": 0.019280831768444688 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.018171526532123665 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.017741691083910972 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.01756128723650169 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.017371048243750575 }, { "system": "ease", "label": "EASEᴿ", "value": 0.01704553328783983 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.017043560978391437 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.0166054602745283 }, { "system": "tiger", "label": "TIGER", "value": 0.016578338730092058 }, { "system": "ials", "label": "iALS", "value": 0.016567610824860482 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.016447176531529106 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.01568193598255609 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.015429249448348082 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.015363445361626942 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.0152211829969402 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.014031346844359584 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.013847837012540538 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.013097507249697103 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.012425180997003491 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.011444757292187186 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.010508457592752492 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.010401254878657003 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.01020438066722946 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.010117304679510008 }, { "system": "popularity", "label": "Popularity", "value": 0.009959300067741882 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.009038373925650557 }, { "system": "content", "label": "Content TF-IDF", "value": 0.001306569213365825 } ] }, "kuairec": { "winner": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.07363896005949103, "runner_up": "recency_pop", "runner_up_value": 0.07026721652772842, "relative_gap": 0.047984589377210715, "ranking": [ { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.07363896005949103 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.07026721652772842 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.06234622409351161 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.058135332522330926 }, { "system": "sasrec", "label": "SASRec", "value": 0.05776940855528339 }, { "system": "popularity", "label": "Popularity", "value": 0.056672023900436005 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.05652702143136791 }, { "system": "content", "label": "Content TF-IDF", "value": 0.053499316791794524 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.05031381716439708 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.04852452028052566 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.048370056753421105 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.047516735554230666 }, { "system": "ease", "label": "EASEᴿ", "value": 0.04696692282777333 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.04554186107547992 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.045000869343417606 }, { "system": "tiger", "label": "TIGER", "value": 0.04350969864523755 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.041461863147960104 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.03864000848925766 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.03773092615926403 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.03703181393861229 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.03611708038429103 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.03140163828925673 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.025199741324654856 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.024277345023105194 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.015442283065724016 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.014513445760685743 }, { "system": "ials", "label": "iALS", "value": 0.011003769182410693 } ] }, "lastfm-1k": { "winner": "lightgcn", "label": "LightGCN", "value": 0.08463504971832904, "runner_up": "vsknn", "runner_up_value": 0.07637357248857803, "relative_gap": 0.10817193645074741, "ranking": [ { "system": "lightgcn", "label": "LightGCN", "value": 0.08463504971832904 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.07637357248857803 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.07441589819843293 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.07124973334513691 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.07095760444205813 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.06957820228628347 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.06779620650707699 }, { "system": "popularity", "label": "Popularity", "value": 0.06212193825712727 }, { "system": "ials", "label": "iALS", "value": 0.053515420036759465 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.04842940509337828 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.045469937188655586 }, { "system": "tiger", "label": "TIGER", "value": 0.04064659006463302 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.03697428806428533 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.03604613457205718 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.03420498950012055 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.03149935228572283 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.030345357454124377 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.028413130974520584 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.027621255561021556 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.023601024162656158 }, { "system": "sasrec", "label": "SASRec", "value": 0.01826701108795635 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.016432880318173377 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.013192091567221008 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.012986182345844645 }, { "system": "content", "label": "Content TF-IDF", "value": 0.004348255033960462 } ] }, "mind-small": { "winner": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.20642916323439603, "runner_up": "vsknn", "runner_up_value": 0.14556992934035518, "relative_gap": 0.41807558861793953, "ranking": [ { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.20642916323439603 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.14556992934035518 }, { "system": "sasrec", "label": "SASRec", "value": 0.13670571220013594 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.1361313529553007 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.13246719352972047 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.13101718775993612 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.12821163107495753 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.12733056999086517 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.12363828298352453 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.12329160115494475 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.12244048310172888 }, { "system": "tiger", "label": "TIGER", "value": 0.12239382800317666 }, { "system": "ease", "label": "EASEᴿ", "value": 0.12076750185438781 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.11595377169132622 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.11373839590501995 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.11203582438819258 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.10650362550858743 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.10438605537411773 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.09986625336312921 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.09503936882658037 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.09272384774413726 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.08342046283070279 }, { "system": "ials", "label": "iALS", "value": 0.06359329382745764 }, { "system": "popularity", "label": "Popularity", "value": 0.048280806968302635 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.04670419644229557 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.03597409228044063 }, { "system": "content", "label": "Content TF-IDF", "value": 0.00444615289904792 } ] }, "ml-1m": { "winner": "itemknn", "label": "ItemKNN", "value": 0.19800911330435214, "runner_up": "itemknn_mlp", "runner_up_value": 0.19783511931226513, "relative_gap": 0.00087948991408534, "ranking": [ { "system": "itemknn", "label": "ItemKNN", "value": 0.19800911330435214 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.19783511931226513 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.19666379900907097 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.19244163508043455 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.18097750457449588 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.17847317491188702 }, { "system": "ease", "label": "EASEᴿ", "value": 0.17818179055618996 }, { "system": "popularity", "label": "Popularity", "value": 0.1780508548957317 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.16571785997436156 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.16541596726998367 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.1634386714194863 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.14617610042627674 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.14084919060222872 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.13055758012588548 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.12911785313248206 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.12382223588911741 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.12088657243635628 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.11919018755690701 }, { "system": "ials", "label": "iALS", "value": 0.11803415224166319 }, { "system": "tiger", "label": "TIGER", "value": 0.11651975859799317 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.11639626084577111 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.11543462968191048 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.11077208808267386 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.10768043915663982 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.10544635768927904 }, { "system": "sasrec", "label": "SASRec", "value": 0.10174682483797279 }, { "system": "content", "label": "Content TF-IDF", "value": 0.025895379210304905 } ] }, "ml-32m": { "winner": "ease", "label": "EASEᴿ", "value": 0.12429915027050366, "runner_up": "recency_pop", "runner_up_value": 0.11468810692943299, "relative_gap": 0.08380156930294698, "ranking": [ { "system": "ease", "label": "EASEᴿ", "value": 0.12429915027050366 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.11468810692943299 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.11378392919622637 }, { "system": "sasrec", "label": "SASRec", "value": 0.11329141971602254 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.10724442100904234 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.10604617630250883 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.1034948361705291 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.10237952654632827 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.1000088159868341 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.0987214117618008 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.09739544191662376 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.0969056321428226 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.09525083118002384 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.094872409834551 }, { "system": "ials", "label": "iALS", "value": 0.09222174214573883 }, { "system": "tiger", "label": "TIGER", "value": 0.09029930154406028 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.08801626201811587 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.08716014338206457 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.0857927821154029 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.08571261303772884 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.08150093108717563 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.07496737475319083 }, { "system": "popularity", "label": "Popularity", "value": 0.06514838633521539 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.06358513640024253 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.06194211452631475 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.058652825930861914 }, { "system": "content", "label": "Content TF-IDF", "value": 0.011742982650376753 } ] }, "retailrocket": { "winner": "ease", "label": "EASEᴿ", "value": 0.028593002967237727, "runner_up": "itemknn", "runner_up_value": 0.027601345076284232, "relative_gap": 0.035927882797478314, "ranking": [ { "system": "ease", "label": "EASEᴿ", "value": 0.028593002967237727 }, { "system": "itemknn", "label": "ItemKNN", "value": 0.027601345076284232 }, { "system": "ials", "label": "iALS", "value": 0.026771809232232817 }, { "system": "vsknn", "label": "VS-KNN", "value": 0.02570415854739308 }, { "system": "itemknn_multitask", "label": "ItemKNN + Multi-task ranker", "value": 0.019124202041535896 }, { "system": "lightgcn", "label": "LightGCN", "value": 0.01741281141911592 }, { "system": "itemknn_mlp", "label": "ItemKNN + MLP ranker", "value": 0.01698501161662412 }, { "system": "sasrec", "label": "SASRec", "value": 0.013681920690353264 }, { "system": "onerec_style", "label": "OneRec-style session generator", "value": 0.01293103448275862 }, { "system": "hstu", "label": "HSTU (Generative Recommenders)", "value": 0.012256667323697108 }, { "system": "option_full", "label": "Option-Attention Scorer (full catalogue)", "value": 0.0120844811894619 }, { "system": "vsknn_gbdt", "label": "VS-KNN + GBDT", "value": 0.012062856237296502 }, { "system": "recency_pop", "label": "Recency-Weighted Popularity", "value": 0.011986034913725015 }, { "system": "twotower_rankmixer", "label": "Two-Tower + RankMixer-style ranker", "value": 0.011910850954338106 }, { "system": "plum_style", "label": "PLUM-style SID generative retrieval", "value": 0.011130597859511674 }, { "system": "option_cascade", "label": "Cheap full pass + Option-Attention reranker", "value": 0.010757502793475805 }, { "system": "twotower_dcnv2", "label": "Two-Tower + DCN-v2", "value": 0.01074398610983388 }, { "system": "bert4rec", "label": "BERT4Rec", "value": 0.009641935923681257 }, { "system": "netflix_foundation_style", "label": "Netflix-style foundation model", "value": 0.00854946890661663 }, { "system": "tiger", "label": "TIGER", "value": 0.008408012890704376 }, { "system": "ease_lambdamart", "label": "EASEᴿ + LambdaMART", "value": 0.006733169078774423 }, { "system": "gru4rec", "label": "GRU4Rec", "value": 0.0052857516617711925 }, { "system": "option_rerank", "label": "ItemKNN + Option-Attention reranker", "value": 0.004810860352094985 }, { "system": "popularity", "label": "Popularity", "value": 0.0030925788817201625 }, { "system": "phoenix_style", "label": "Phoenix-style unified transformer", "value": 0.003018519844685174 }, { "system": "option_calibrated", "label": "Calibrated Multi-Outcome Option Scorer", "value": 0.002719524799876972 }, { "system": "content", "label": "Content TF-IDF", "value": 0 } ] } }, "randomised_exposure": { "signal": "y_click", "n_logged_rows": 9091, "n_random_rows": 70339, "note": "ground truth is a plain average of the reward over the randomly-exposed impressions the system's own slate intersects; it needs no propensity model, which is what makes it ground truth", "ranking_agreement": { "n_systems": 23, "spearman_offline_vs_random_exposure": 0.1689723320158103, "offline_best": "sasrec", "random_exposure_best": "lightgcn", "same_winner": false, "offline_metric": "ndcg@10 on held-out engagements", "ground_truth_metric": "mean y_click over randomly-exposed impressions the slate intersects" } }, "datasets": [ { "name": "amazon-beauty-2023", "axis": "e-commerce reviews; very sparse, very long-tailed, rich item text", "n_users": 24979, "n_items": 15588, "n_inter": 204659, "gini": 0.47527449253522747, "median_hist": 6, "impressions": false, "licence": "Amazon Reviews 2023 (McAuley Lab, UCSD) — research use; see dataset page.", "licence_url": "https://amazon-reviews-2023.github.io/" }, { "name": "goodreads-poetry", "axis": "books; medium catalogue, slow-moving, strong content signal", "n_users": 5791, "n_items": 2903, "n_inter": 67954, "gini": 0.6470310514251432, "median_hist": 7, "impressions": false, "licence": "Goodreads datasets (UCSD McAuley Lab) — academic use only; redistribution not permitted.", "licence_url": "https://mengtingwan.github.io/data/goodreads.html" }, { "name": "hm-fashion", "axis": "fashion e-commerce; seasonal catalogue turnover and heavy supplier concentration", "n_users": 32343, "n_items": 39979, "n_inter": 1053294, "gini": 0.5491414909159775, "median_hist": 16, "impressions": false, "licence": "H&M Personalized Fashion Recommendations (Kaggle competition data) — competition rules apply; research use.", "licence_url": "https://www.kaggle.com/competitions/h-and-m-personalized-fashion-recommendations" }, { "name": "kuairand-pure", "axis": "short-video feed; impression logs; eight engagement signals; a randomised-exposure slice", "n_users": 25103, "n_items": 7539, "n_inter": 863799, "gini": 0.65767437715805, "median_hist": 23, "impressions": true, "licence": "KuaiRand (Gao et al., CIKM 2022) — CC BY 4.0.", "licence_url": "https://kuairand.com/" }, { "name": "kuairec", "axis": "short-video feed; near-fully observed user-item matrix, so exposure bias can be measured directly", "n_users": 1341, "n_items": 1821, "n_inter": 84911, "gini": 0.5426201147077812, "median_hist": 33, "impressions": true, "licence": "KuaiRec (Gao et al., CIKM 2022) — CC BY 4.0.", "licence_url": "https://kuairec.com/" }, { "name": "lastfm-1k", "axis": "music; extremely long, repetitive sequences — the regime sequential models were designed for", "n_users": 761, "n_items": 42638, "n_inter": 13310148, "gini": 0.7206761211578896, "median_hist": 9016, "impressions": false, "licence": "Last.fm-1K (Celma, 2010) — research use. Mirror hosted on Hugging Face.", "licence_url": "http://ocelma.net/MusicRecommendationDataset/lastfm-1K.html" }, { "name": "mind-small", "axis": "news; impressions; a catalogue that turns over in days, so almost every target item is cold", "n_users": 27095, "n_items": 3711, "n_inter": 171444, "gini": 0.7188395963114844, "median_hist": 4, "impressions": true, "licence": "Microsoft MIND — Microsoft Research License Terms, research use only.", "licence_url": "https://msnews.github.io/" }, { "name": "ml-1m", "axis": "movies; small catalogue, dense histories; the field's most-reported dataset", "n_users": 5970, "n_items": 3125, "n_inter": 574376, "gini": 0.6685062215318307, "median_hist": 54, "impressions": false, "licence": "GroupLens / MovieLens Terms of Use — research use permitted, redistribution of the dataset not permitted without permission.", "licence_url": "https://files.grouplens.org/datasets/movielens/ml-1m-README.txt" }, { "name": "ml-32m", "axis": "movies; large catalogue, long time span, sparse per-user coverage", "n_users": 35073, "n_items": 10476, "n_inter": 3093573, "gini": 0.7992707739291152, "median_hist": 43, "impressions": false, "licence": "GroupLens / MovieLens Terms of Use — research use permitted, redistribution of the dataset not permitted without permission.", "licence_url": "https://files.grouplens.org/datasets/movielens/ml-32m-README.html" }, { "name": "retailrocket", "axis": "e-commerce; sparse sessions, large catalogue, strong recency", "n_users": 1499, "n_items": 2156, "n_inter": 19062, "gini": 0.40496836563078853, "median_hist": 7, "impressions": false, "licence": "RetailRocket e-commerce dataset — CC BY-NC-SA 4.0.", "licence_url": "https://www.kaggle.com/datasets/retailrocket/ecommerce-dataset" } ], "stage_boards": { "retrieval": { "popularity": { "recall@100": 0.14274299332149334, "recall@500": 0.3005263015322598, "recall@1000": 0.40121424413553675, "coverage": 0.052043952579925154, "tail_recall@100": 0 }, "itemknn": { "recall@100": 0.15841987619411263, "recall@500": 0.3159976004818723, "recall@1000": 0.41048513563921407, "coverage": 0.43487265184192986, "tail_recall@100": 0.012143398366666763 }, "recency_pop": { "recall@100": 0.19384892477679774, "recall@500": 0.3468908887311074, "recall@1000": 0.4317839890966309, "coverage": 0.048126110645514905, "tail_recall@100": 0.0033218831413844934 }, "ease": { "recall@100": 0.20821024393646054, "recall@500": 0.37685394712671305, "recall@1000": 0.4647594165965907, "coverage": 0.651947385149727, "tail_recall@100": 0.019444912487324648 }, "ials": { "recall@100": 0.12902717705003044, "recall@500": 0.2593408365047947, "recall@1000": 0.3865310599279166, "coverage": 0.6819505057800899, "tail_recall@100": 0.028952942412104936 }, "vsknn": { "recall@100": 0.1817850328838331, "recall@500": 0.3382592663875476, "recall@1000": 0.4175984795080036, "coverage": 0.4271929794823383, "tail_recall@100": 0.012621185722408123 }, "content": { "recall@100": 0.07738399148943434, "recall@500": 0.21194447733230065, "recall@1000": 0.3137793898703018, "coverage": 0.5045029600760808, "tail_recall@100": 0.0887381763157607 }, "itemknn_mlp": { "recall@100": 0.1584198761941126, "recall@500": 0.31599760048187225, "recall@1000": 0.41048513563921407, "coverage": 0.4348726518419298, "tail_recall@100": 0.012143398366666763 }, "itemknn_multitask": { "recall@100": 0.1584198761941126, "recall@500": 0.31599760048187225, "recall@1000": 0.41048513563921407, "coverage": 0.4348726518419298, "tail_recall@100": 0.012143398366666763 }, "ease_lambdamart": { "recall@100": 0.20821024393646054, "recall@500": 0.37685394712671305, "recall@1000": 0.4647594165965907, "coverage": 0.651947385149727, "tail_recall@100": 0.019444912487324648 }, "vsknn_gbdt": { "recall@100": 0.1817850328838331, "recall@500": 0.3382592663875476, "recall@1000": 0.4175984795080036, "coverage": 0.4271929794823383, "tail_recall@100": 0.012621185722408123 }, "twotower_dcnv2": { "recall@100": 0.14965444285077184, "recall@500": 0.3608633282685304, "recall@1000": 0.4861469157143906, "coverage": 0.29231409676877207, "tail_recall@100": 0.004975757601298853 }, "twotower_rankmixer": { "recall@100": 0.14965444285077184, "recall@500": 0.3608633282685304, "recall@1000": 0.4861469157143906, "coverage": 0.29231409676877207, "tail_recall@100": 0.004975757601298853 }, "lightgcn": { "recall@100": 0.16411816511728064, "recall@500": 0.332110020815651, "recall@1000": 0.42800172246607965, "coverage": 0.6669495694452565, "tail_recall@100": 0.01902461407240005 }, "gru4rec": { "recall@100": 0.15328176384917075, "recall@500": 0.31606425359300483, "recall@1000": 0.4155442380166933, "coverage": 0.780563810036284, "tail_recall@100": 0.014886377228485912 }, "sasrec": { "recall@100": 0.178106046221883, "recall@500": 0.33944563478423784, "recall@1000": 0.432402012896911, "coverage": 0.5400026910053353, "tail_recall@100": 0.019819515076015762 }, "bert4rec": { "recall@100": 0.17372499806588007, "recall@500": 0.33463654541498256, "recall@1000": 0.42498751887318614, "coverage": 0.3785427227002683, "tail_recall@100": 0.01505917491173919 }, "hstu": { "recall@100": 0.14714749553166523, "recall@500": 0.29698583730092765, "recall@1000": 0.3933680906704879, "coverage": 0.5578899049272978, "tail_recall@100": 0.009566561686443957 }, "netflix_foundation_style": { "recall@100": 0.1459713779459058, "recall@500": 0.312159827166955, "recall@1000": 0.423557421190913, "coverage": 0.5739925843689451, "tail_recall@100": 0.015740374001096274 }, "phoenix_style": { "recall@100": 0.03611701902065966, "recall@500": 0.14719372755107032, "recall@1000": 0.25640881282263456, "coverage": 0.08466757310651513, "tail_recall@100": 0.002196360608761282 }, "option_full": { "recall@100": 0.16016210561913946, "recall@500": 0.323993336465064, "recall@1000": 0.41463539028372365, "coverage": 0.4709955999124393, "tail_recall@100": 0.007143330595608963 }, "option_rerank": { "recall@100": 0.1584198761941126, "recall@500": 0.31599760048187225, "recall@1000": 0.41048513563921407, "coverage": 0.4348726518419298, "tail_recall@100": 0.012143398366666763 }, "option_cascade": { "recall@100": 0.1637260884908064, "recall@500": 0.3315647057468011, "recall@1000": 0.42151749366169905, "coverage": 0.4028982396608442, "tail_recall@100": 0.005091219438057601 }, "option_calibrated": { "recall@100": 0.15868722520277906, "recall@500": 0.3239408176966857, "recall@1000": 0.41504326052852203, "coverage": 0.48763172393007165, "tail_recall@100": 0.00812810968459075 } }, "ranking": { "itemknn_mlp": { "ndcg@10": 0.06806640850716633, "mrr@10": 0.11220087460805894, "recall@20": 0.07739334321383459 }, "itemknn_multitask": { "ndcg@10": 0.0694144505795006, "mrr@10": 0.11340993755090752, "recall@20": 0.07883341183842087 }, "ease_lambdamart": { "ndcg@10": 0.05294542838722742, "mrr@10": 0.08979565200453873, "recall@20": 0.07682393492978642 }, "vsknn_gbdt": { "ndcg@10": 0.055420470400156165, "mrr@10": 0.08917288362668539, "recall@20": 0.07591992403361116 }, "twotower_dcnv2": { "ndcg@10": 0.04631766607849679, "mrr@10": 0.08127993939213482, "recall@20": 0.05684720363888712 }, "twotower_rankmixer": { "ndcg@10": 0.05016253168640431, "mrr@10": 0.0876914058297529, "recall@20": 0.05861622342619473 }, "lightgcn": { "ndcg@10": 0.06322828062908421, "mrr@10": 0.10314007349278957, "recall@20": 0.07269719023532037 }, "gru4rec": { "ndcg@10": 0.05369919398713745, "mrr@10": 0.09173227652829166, "recall@20": 0.07038550370713992 }, "sasrec": { "ndcg@10": 0.060300068536714026, "mrr@10": 0.10047505365473748, "recall@20": 0.08181243276217964 }, "bert4rec": { "ndcg@10": 0.058256059288303484, "mrr@10": 0.09846277933339831, "recall@20": 0.0772554946984501 }, "hstu": { "ndcg@10": 0.05236508594982897, "mrr@10": 0.08979659187940206, "recall@20": 0.06791265124868004 }, "netflix_foundation_style": { "ndcg@10": 0.04802700890222727, "mrr@10": 0.0795615916683015, "recall@20": 0.0636760126639733 }, "phoenix_style": { "ndcg@10": 0.060290959271488906, "mrr@10": 0.09641751443702105, "recall@20": 0.07777689663544202 }, "option_full": { "ndcg@10": 0.06824009229273888, "mrr@10": 0.11496838332587919, "recall@20": 0.08636926462568409 }, "option_rerank": { "ndcg@10": 0.05272811682522757, "mrr@10": 0.09363198289155125, "recall@20": 0.0661107104183058 }, "option_cascade": { "ndcg@10": 0.07005531610320892, "mrr@10": 0.11741724541245854, "recall@20": 0.08628806157558758 }, "option_calibrated": { "ndcg@10": 0.06688574222914205, "mrr@10": 0.11157751895724458, "recall@20": 0.08655697559881204 } }, "batch_invariance": { "itemknn_mlp": { "datasets_tested": 10, "invariant_on_all": false }, "itemknn_multitask": { "datasets_tested": 10, "invariant_on_all": false }, "ease_lambdamart": { "datasets_tested": 8, "invariant_on_all": false }, "vsknn_gbdt": { "datasets_tested": 10, "invariant_on_all": false }, "twotower_dcnv2": { "datasets_tested": 10, "invariant_on_all": false }, "twotower_rankmixer": { "datasets_tested": 10, "invariant_on_all": false }, "lightgcn": { "datasets_tested": 10, "invariant_on_all": true }, "gru4rec": { "datasets_tested": 10, "invariant_on_all": false }, "sasrec": { "datasets_tested": 10, "invariant_on_all": true }, "bert4rec": { "datasets_tested": 10, "invariant_on_all": true }, "hstu": { "datasets_tested": 10, "invariant_on_all": true }, "netflix_foundation_style": { "datasets_tested": 10, "invariant_on_all": true }, "phoenix_style": { "datasets_tested": 10, "invariant_on_all": true }, "option_full": { "datasets_tested": 8, "invariant_on_all": true }, "option_rerank": { "datasets_tested": 8, "invariant_on_all": true }, "option_cascade": { "datasets_tested": 8, "invariant_on_all": true }, "option_calibrated": { "datasets_tested": 8, "invariant_on_all": true } }, "unavailable": { "tiger": { "amazon-beauty-2023": "no separable ranking stage", "goodreads-poetry": "no separable ranking stage", "hm-fashion": "no separable ranking stage", "kuairand-pure": "no separable ranking stage", "kuairec": "no separable ranking stage", "lastfm-1k": "no separable ranking stage", "mind-small": "no separable ranking stage", "ml-1m": "no separable ranking stage", "ml-32m": "no separable ranking stage", "retailrocket": "no separable ranking stage" }, "plum_style": { "amazon-beauty-2023": "no separable ranking stage", "goodreads-poetry": "no separable ranking stage", "hm-fashion": "no separable ranking stage", "kuairand-pure": "no separable ranking stage", "kuairec": "no separable ranking stage", "lastfm-1k": "no separable ranking stage", "mind-small": "no separable ranking stage", "ml-1m": "no separable ranking stage", "ml-32m": "no separable ranking stage", "retailrocket": "no separable ranking stage" }, "onerec_style": { "amazon-beauty-2023": "no separable ranking stage", "goodreads-poetry": "no separable ranking stage", "hm-fashion": "no separable ranking stage", "kuairand-pure": "no separable ranking stage", "kuairec": "no separable ranking stage", "lastfm-1k": "no separable ranking stage", "mind-small": "no separable ranking stage", "ml-1m": "no separable ranking stage", "ml-32m": "no separable ranking stage", "retailrocket": "no separable ranking stage" }, "popularity": { "amazon-beauty-2023": "no separable ranking stage", "goodreads-poetry": "no separable ranking stage", "hm-fashion": "no separable ranking stage", "kuairand-pure": "no separable ranking stage", "kuairec": "no separable ranking stage", "lastfm-1k": "no separable ranking stage", "mind-small": "no separable ranking stage", "ml-1m": "no separable ranking stage", "ml-32m": "no separable ranking stage", "retailrocket": "no separable ranking stage" }, "itemknn": { "amazon-beauty-2023": "no separable ranking stage", "goodreads-poetry": "no separable ranking stage", "hm-fashion": "no separable ranking stage", "kuairand-pure": "no separable ranking stage", "kuairec": "no separable ranking stage", "lastfm-1k": "no separable ranking stage", "mind-small": "no separable ranking stage", "ml-1m": "no separable ranking stage", "ml-32m": "no separable ranking stage", "retailrocket": "no separable ranking stage" }, "recency_pop": { "amazon-beauty-2023": "no separable ranking stage", "goodreads-poetry": "no separable ranking stage", "hm-fashion": "no separable ranking stage", "kuairand-pure": "no separable ranking stage", "kuairec": "no separable ranking stage", "lastfm-1k": "no separable ranking stage", "mind-small": "no separable ranking stage", "ml-1m": "no separable ranking stage", "ml-32m": "no separable ranking stage", "retailrocket": "no separable ranking stage" }, "ease": { "amazon-beauty-2023": "no separable ranking stage", "goodreads-poetry": "no separable ranking stage", "kuairand-pure": "no separable ranking stage", "kuairec": "no separable ranking stage", "mind-small": "no separable ranking stage", "ml-1m": "no separable ranking stage", "ml-32m": "no separable ranking stage", "retailrocket": "no separable ranking stage" }, "ials": { "amazon-beauty-2023": "no separable ranking stage", "goodreads-poetry": "no separable ranking stage", "hm-fashion": "no separable ranking stage", "kuairand-pure": "no separable ranking stage", "kuairec": "no separable ranking stage", "lastfm-1k": "no separable ranking stage", "mind-small": "no separable ranking stage", "ml-1m": "no separable ranking stage", "ml-32m": "no separable ranking stage", "retailrocket": "no separable ranking stage" }, "vsknn": { "amazon-beauty-2023": "no separable ranking stage", "goodreads-poetry": "no separable ranking stage", "hm-fashion": "no separable ranking stage", "kuairand-pure": "no separable ranking stage", "kuairec": "no separable ranking stage", "lastfm-1k": "no separable ranking stage", "mind-small": "no separable ranking stage", "ml-1m": "no separable ranking stage", "ml-32m": "no separable ranking stage", "retailrocket": "no separable ranking stage" }, "content": { "amazon-beauty-2023": "no separable ranking stage", "goodreads-poetry": "no separable ranking stage", "hm-fashion": "no separable ranking stage", "kuairand-pure": "no separable ranking stage", "kuairec": "no separable ranking stage", "lastfm-1k": "no separable ranking stage", "mind-small": "no separable ranking stage", "ml-1m": "no separable ranking stage", "ml-32m": "no separable ranking stage", "retailrocket": "no separable ranking stage" }, "option_full": { "amazon-beauty-2023": "failed: OutOfMemoryError('CUDA out of memory. Tried to allocate 22.53 GiB. GPU 0 has a total capacity of 22.06 GiB of which 20.92 GiB is free. Including non-PyTorch memory, this process has 1.13 GiB memory in use. Of the allocated memory 312.26 MiB is allocated by PyTorch, and 531.74 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)')", "hm-fashion": "failed: OutOfMemoryError('CUDA out of memory. Tried to allocate 45.75 GiB. GPU 0 has a total capacity of 22.06 GiB of which 21.22 GiB is free. Including non-PyTorch memory, this process has 850.00 MiB memory in use. Of the allocated memory 459.53 MiB is allocated by PyTorch, and 78.47 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)')" }, "option_rerank": { "amazon-beauty-2023": "failed: OutOfMemoryError('CUDA out of memory. Tried to allocate 22.53 GiB. GPU 0 has a total capacity of 22.06 GiB of which 20.92 GiB is free. Including non-PyTorch memory, this process has 1.13 GiB memory in use. Of the allocated memory 309.93 MiB is allocated by PyTorch, and 534.07 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)')", "hm-fashion": "failed: OutOfMemoryError('CUDA out of memory. Tried to allocate 45.75 GiB. GPU 0 has a total capacity of 22.06 GiB of which 21.13 GiB is free. Including non-PyTorch memory, this process has 948.00 MiB memory in use. Of the allocated memory 457.92 MiB is allocated by PyTorch, and 178.08 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)')" }, "option_cascade": { "amazon-beauty-2023": "failed: OutOfMemoryError('CUDA out of memory. Tried to allocate 22.53 GiB. GPU 0 has a total capacity of 22.06 GiB of which 20.92 GiB is free. Including non-PyTorch memory, this process has 1.13 GiB memory in use. Of the allocated memory 312.26 MiB is allocated by PyTorch, and 531.74 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)')", "hm-fashion": "failed: OutOfMemoryError('CUDA out of memory. Tried to allocate 45.75 GiB. GPU 0 has a total capacity of 22.06 GiB of which 21.26 GiB is free. Including non-PyTorch memory, this process has 810.00 MiB memory in use. Of the allocated memory 460.21 MiB is allocated by PyTorch, and 37.79 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)')" }, "option_calibrated": { "amazon-beauty-2023": "failed: OutOfMemoryError('CUDA out of memory. Tried to allocate 22.53 GiB. GPU 0 has a total capacity of 22.06 GiB of which 20.92 GiB is free. Including non-PyTorch memory, this process has 1.13 GiB memory in use. Of the allocated memory 312.26 MiB is allocated by PyTorch, and 531.74 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)')", "hm-fashion": "failed: OutOfMemoryError('CUDA out of memory. Tried to allocate 45.75 GiB. GPU 0 has a total capacity of 22.06 GiB of which 21.26 GiB is free. Including non-PyTorch memory, this process has 810.00 MiB memory in use. Of the allocated memory 460.21 MiB is allocated by PyTorch, and 37.79 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)')" } } }, "robustness": { "percentile_scores": { "bert4rec": 54.999323933147515, "content": 32.09641665524021, "ease": 65.12161252314421, "ease_lambdamart": 48.19264433597034, "gru4rec": 48.30959481694781, "hstu": 44.87231858555394, "ials": 50.635836113777366, "itemknn": 57.54152574005523, "itemknn_mlp": 66.38943164678466, "itemknn_multitask": 68.8247006261713, "lightgcn": 60.321493593552475, "netflix_foundation_style": 42.465996693937925, "onerec_style": 38.417022792022884, "option_calibrated": 42.379726754726796, "option_cascade": 49.58298101680462, "option_full": 45.69283521489406, "option_rerank": 36.38548951048954, "phoenix_style": 20.484778021542738, "plum_style": 54.50579513079518, "popularity": 36.275366790072745, "recency_pop": 59.414278531925625, "sasrec": 60.6967583070525, "tiger": 48.88558473117308, "twotower_dcnv2": 49.311697289638516, "twotower_rankmixer": 47.16570874659114, "vsknn": 64.41588776147606, "vsknn_gbdt": 59.08343426725783 }, "rank_correlation_frozen_vs_live_percentile": 0.9829059829059829, "like_for_like": { "datasets": [ "amazon-beauty-2023", "goodreads-poetry", "kuairand-pure", "kuairec", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "order": [ "itemknn_multitask", "itemknn_mlp", "sasrec", "ease", "vsknn", "bert4rec", "recency_pop", "vsknn_gbdt", "lightgcn", "itemknn", "plum_style", "gru4rec", "option_cascade", "twotower_dcnv2", "tiger", "hstu", "option_full", "twotower_rankmixer", "ials", "ease_lambdamart", "option_calibrated", "netflix_foundation_style", "onerec_style", "option_rerank", "popularity", "content", "phoenix_style" ], "index": { "itemknn_multitask": 69.18678830227744, "itemknn_mlp": 67.37729684265011, "sasrec": 66.1136451863354, "ease": 64.51358695652173, "vsknn": 63.630790631469985, "bert4rec": 60.61455098343684, "recency_pop": 60.40301501035196, "vsknn_gbdt": 59.96703545548654, "lightgcn": 57.926533385093165, "itemknn": 56.23369565217391, "plum_style": 55.716906055900616, "gru4rec": 54.37247670807453, "option_cascade": 53.01950698757764, "twotower_dcnv2": 52.16755628881988, "tiger": 52.023033126293996, "hstu": 51.616330227743276, "option_full": 51.5875873447205, "twotower_rankmixer": 51.281719073498955, "ials": 50.99980590062111, "ease_lambdamart": 50.875905797101446, "option_calibrated": 48.23722179089027, "netflix_foundation_style": 44.642145445134574, "onerec_style": 40.833753881987576, "option_rerank": 38.66409484989648, "popularity": 35.24915890269151, "content": 35.071800595238095, "phoenix_style": 26.042168090062113 }, "rank_correlation_with_headline": 0.9792429792429792 }, "note": "the headline index scores each measurement against a frozen field; the robustness column scores it against the live one, which depends on no published constant. A high rank correlation means the ordering is a property of the results rather than of the frozen constants" }, "coverage": { "bert4rec": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "content": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "ease": [ "amazon-beauty-2023", "goodreads-poetry", "kuairand-pure", "kuairec", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "ease_lambdamart": [ "amazon-beauty-2023", "goodreads-poetry", "kuairand-pure", "kuairec", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "gru4rec": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "hstu": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "ials": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "itemknn": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "itemknn_mlp": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "itemknn_multitask": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "lightgcn": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "netflix_foundation_style": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "onerec_style": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "option_calibrated": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "option_cascade": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "option_full": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "option_rerank": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "phoenix_style": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "plum_style": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "popularity": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "recency_pop": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "sasrec": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "tiger": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "twotower_dcnv2": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "twotower_rankmixer": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "vsknn": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ], "vsknn_gbdt": [ "amazon-beauty-2023", "goodreads-poetry", "hm-fashion", "kuairand-pure", "kuairec", "lastfm-1k", "mind-small", "ml-1m", "ml-32m", "retailrocket" ] }, "skipped": { "ease": { "hm-fashion": "catalogue has 39979 items; this system is not attempted above 25000 by policy", "lastfm-1k": "catalogue has 42638 items; this system is not attempted above 25000 by policy" }, "ease_lambdamart": { "hm-fashion": "catalogue has 39979 items; this system is not attempted above 25000 by policy", "lastfm-1k": "catalogue has 42638 items; this system is not attempted above 25000 by policy" } }, "confidence_levels": { "official": "Authors' released implementation, run as published.", "reference": "Faithful implementation of a fully specified public method.", "reproduction": "Our implementation from the paper, using the paper's stated design.", "proxy": "Architecture proxy. The target system is proprietary and only partially described; we implement the published mechanism at benchmark scale. Not the production system." }, "caveats": [ "On the one dataset that inserts randomly-chosen items into real feeds — the only place an unbiased estimate is available — the offline ordering and the unbiased ordering correlate at 0.17 and name different winners. That is a limit on what any offline recommender benchmark, including this one, can tell you.", "Public offline datasets are not production environments. Offline relevance is a proxy for user value, and the mapping between them is platform-specific.", "Several entrants are architecture proxies for proprietary systems. They implement a published mechanism at benchmark scale and are not those companies' production systems.", "No single recommender wins every scenario. The winner changes 5 ways across the scenario boards.", "Entrants added after the calibrating release are measured in their own run rather than beside the systems they are ranked against. Every board is unaffected — a score is a percentile in a frozen distribution, not a rank against whoever ran that day — but a head-to-head significance cell exists only where both systems' result files carry per-user vectors. Which cells those are is published in `pairwise_provenance`. Cost and latency use the same instance types across runs, not the same job.", "Serving latency is offline batch throughput on the benchmark host, not a production p99. Multi-stage systems additionally build their ranker's features online here, which a deployed pipeline precomputes — so their measured serving cost is an upper bound and the cost gap to single-stage systems is overstated.", "Scale is the untested variable: the central claim of the industrial generative systems is a scaling claim this benchmark is too small to test.", "The index scores each measurement by its percentile in a frozen, published distribution. That records where a system placed, not by how much: doubling the best cold-start result and beating it by a hair score the same. Raw metric values are published beside every score, and the scenario boards are stated in raw metrics.", "This is index version v3.1 and its scores are not comparable to v3.0's. The v3.0 scale rescaled every metric so Popularity read 0 and ItemKNN 100; it was withdrawn before publication because on two of ten datasets Popularity beat ItemKNN, which inverted the scale, and where the two were close a worse result could reach -381. The effect was to flatter ItemKNN by about six places. Under v3.1 nothing is pinned and ItemKNN places 9th." ] }