{
  "study": "Yambda-50M likes temporal baselines",
  "quality": {
    "rows": 881456,
    "users": 8283,
    "items": 181304,
    "min_ts": 80,
    "max_ts": 25999970,
    "null_rows": 0,
    "invalid_flags": 0,
    "exact_duplicates": 6405,
    "schema": [
      [
        "uid",
        "UINTEGER",
        "YES",
        null,
        null,
        null
      ],
      [
        "timestamp",
        "UINTEGER",
        "YES",
        null,
        null,
        null
      ],
      [
        "item_id",
        "UINTEGER",
        "YES",
        null,
        null,
        null
      ],
      [
        "is_organic",
        "UTINYINT",
        "YES",
        null,
        null,
        null
      ]
    ],
    "organic_counts": [
      [
        0,
        379255
      ],
      [
        1,
        502201
      ]
    ]
  },
  "cutoffs": [
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "history_rows": 447105,
      "history_users": 6039,
      "catalog_items": 121347,
      "eval_users": 5516,
      "all_future_users": 7760,
      "target_items": 7069,
      "cold_targets": 1528,
      "users_with_tied_targets": 463,
      "top10_overlap_recent_past": 3,
      "top10_overlap_future_past": 6
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "history_rows": 532276,
      "history_users": 6541,
      "catalog_items": 134035,
      "eval_users": 5795,
      "all_future_users": 7537,
      "target_items": 7364,
      "cold_targets": 1515,
      "users_with_tied_targets": 488,
      "top10_overlap_recent_past": 5,
      "top10_overlap_future_past": 8
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "history_rows": 631354,
      "history_users": 7176,
      "catalog_items": 148481,
      "eval_users": 6060,
      "all_future_users": 7167,
      "target_items": 8508,
      "cold_targets": 1734,
      "users_with_tied_targets": 528,
      "top10_overlap_recent_past": 5,
      "top10_overlap_future_past": 9
    }
  ],
  "metrics": [
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "all",
      "method": "past_pop",
      "n": 5516,
      "hits": 135,
      "hit10": 0.024474256707759247,
      "hit10_wilson95": [
        0.02071554851626863,
        0.028894836286196327
      ],
      "recall10": 0.02070223587314578,
      "ndcg10": 0.009124867533001305
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "all",
      "method": "recent_pop",
      "n": 5516,
      "hits": 188,
      "hit10": 0.0340826686004351,
      "hit10_wilson95": [
        0.029609408831536593,
        0.03920442606832422
      ],
      "recall10": 0.030918527035927085,
      "ndcg10": 0.017209515006485435
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "all",
      "method": "future_control",
      "n": 5516,
      "hits": 209,
      "hit10": 0.03788977519941987,
      "hit10_wilson95": [
        0.03316427729079905,
        0.04325847179859598
      ],
      "recall10": 0.034218230681984575,
      "ndcg10": 0.019851722029973207
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "organic",
      "method": "past_pop",
      "n": 3488,
      "hits": 101,
      "hit10": 0.028956422018348624,
      "hit10_wilson95": [
        0.02388877405375448,
        0.03506048273472907
      ],
      "recall10": 0.025708990597963947,
      "ndcg10": 0.01150996498905929
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "organic",
      "method": "recent_pop",
      "n": 3488,
      "hits": 134,
      "hit10": 0.03841743119266055,
      "hit10_wilson95": [
        0.032530064904369736,
        0.04532039366499029
      ],
      "recall10": 0.03514674117701029,
      "ndcg10": 0.02001691444859375
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "organic",
      "method": "future_control",
      "n": 3488,
      "hits": 157,
      "hit10": 0.04501146788990826,
      "hit10_wilson95": [
        0.03861709647098854,
        0.05240692697595206
      ],
      "recall10": 0.041491048255402174,
      "ndcg10": 0.022654735686571555
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "algorithmic",
      "method": "past_pop",
      "n": 1990,
      "hits": 30,
      "hit10": 0.01507537688442211,
      "hit10_wilson95": [
        0.010580097326888156,
        0.02143922825101145
      ],
      "recall10": 0.011931482810879796,
      "ndcg10": 0.0046396822963635145
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "algorithmic",
      "method": "recent_pop",
      "n": 1990,
      "hits": 51,
      "hit10": 0.02562814070351759,
      "hit10_wilson95": [
        0.019545907056740024,
        0.03353828293863534
      ],
      "recall10": 0.023335726250299116,
      "ndcg10": 0.012080117598250723
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "algorithmic",
      "method": "future_control",
      "n": 1990,
      "hits": 48,
      "hit10": 0.024120603015075376,
      "hit10_wilson95": [
        0.018240996304193957,
        0.031833927345827726
      ],
      "recall10": 0.021521097551248303,
      "ndcg10": 0.014912024148242647
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "mixed",
      "method": "past_pop",
      "n": 38,
      "hits": 4,
      "hit10": 0.10526315789473684,
      "hit10_wilson95": [
        0.04170257347553526,
        0.24130521722887316
      ],
      "recall10": 0.020445344129554656,
      "ndcg10": 0.025080096327184102
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "mixed",
      "method": "recent_pop",
      "n": 38,
      "hits": 3,
      "hit10": 0.07894736842105263,
      "hit10_wilson95": [
        0.027214595506987724,
        0.20799371457771454
      ],
      "recall10": 0.039912280701754385,
      "ndcg10": 0.02813824101472943
    },
    {
      "fraction": 0.6,
      "cutoff": 15600014,
      "recent_start": 12480027,
      "group": "mixed",
      "method": "future_control",
      "n": 38,
      "hits": 4,
      "hit10": 0.10526315789473684,
      "hit10_wilson95": [
        0.04170257347553526,
        0.24130521722887316
      ],
      "recall10": 0.031578947368421054,
      "ndcg10": 0.021248752304414892
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "all",
      "method": "past_pop",
      "n": 5795,
      "hits": 134,
      "hit10": 0.023123382226056945,
      "hit10_wilson95": [
        0.01955808704228861,
        0.027320493921016143
      ],
      "recall10": 0.019176679445402696,
      "ndcg10": 0.0081390152122537
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "all",
      "method": "recent_pop",
      "n": 5795,
      "hits": 164,
      "hit10": 0.02830025884383089,
      "hit10_wilson95": [
        0.024333173727437346,
        0.032892301598487585
      ],
      "recall10": 0.02471223965740812,
      "ndcg10": 0.014325167311957255
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "all",
      "method": "future_control",
      "n": 5795,
      "hits": 144,
      "hit10": 0.024849007765314925,
      "hit10_wilson95": [
        0.021144901733782816,
        0.029182644017061995
      ],
      "recall10": 0.02117899641956009,
      "ndcg10": 0.013203101507879069
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "organic",
      "method": "past_pop",
      "n": 3693,
      "hits": 90,
      "hit10": 0.024370430544272948,
      "hit10_wilson95": [
        0.019869578852050956,
        0.02985975384141549
      ],
      "recall10": 0.020913194913262944,
      "ndcg10": 0.00876950451217579
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "organic",
      "method": "recent_pop",
      "n": 3693,
      "hits": 118,
      "hit10": 0.031952342269157864,
      "hit10_wilson95": [
        0.0267485375472121,
        0.038128861576418274
      ],
      "recall10": 0.028567315321242005,
      "ndcg10": 0.015454415228448388
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "organic",
      "method": "future_control",
      "n": 3693,
      "hits": 98,
      "hit10": 0.026536691037097212,
      "hit10_wilson95": [
        0.02182434208287663,
        0.03223300959063666
      ],
      "recall10": 0.023269931134557855,
      "ndcg10": 0.013702574779499897
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "algorithmic",
      "method": "past_pop",
      "n": 2055,
      "hits": 39,
      "hit10": 0.01897810218978102,
      "hit10_wilson95": [
        0.013913771483198382,
        0.025837448053430224
      ],
      "recall10": 0.016036380488935232,
      "ndcg10": 0.006832316749396592
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "algorithmic",
      "method": "recent_pop",
      "n": 2055,
      "hits": 41,
      "hit10": 0.01995133819951338,
      "hit10_wilson95": [
        0.014740847311344472,
        0.026953212448781093
      ],
      "recall10": 0.017899432278994324,
      "ndcg10": 0.012048429634541408
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "algorithmic",
      "method": "future_control",
      "n": 2055,
      "hits": 41,
      "hit10": 0.01995133819951338,
      "hit10_wilson95": [
        0.014740847311344472,
        0.026953212448781093
      ],
      "recall10": 0.017447572703047157,
      "ndcg10": 0.011989523672937496
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "mixed",
      "method": "past_pop",
      "n": 47,
      "hits": 5,
      "hit10": 0.10638297872340426,
      "hit10_wilson95": [
        0.04630478218108157,
        0.22594269343738355
      ],
      "recall10": 0.020035460992907803,
      "ndcg10": 0.01573195896882982
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "mixed",
      "method": "recent_pop",
      "n": 47,
      "hits": 5,
      "hit10": 0.10638297872340426,
      "hit10_wilson95": [
        0.04630478218108157,
        0.22594269343738355
      ],
      "recall10": 0.019680851063829788,
      "ndcg10": 0.025141834790421397
    },
    {
      "fraction": 0.7,
      "cutoff": 18200003,
      "recent_start": 14560018,
      "group": "mixed",
      "method": "future_control",
      "n": 47,
      "hits": 5,
      "hit10": 0.10638297872340426,
      "hit10_wilson95": [
        0.04630478218108157,
        0.22594269343738355
      ],
      "recall10": 0.020035460992907803,
      "ndcg10": 0.027019009139990006
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "all",
      "method": "past_pop",
      "n": 6060,
      "hits": 88,
      "hit10": 0.014521452145214522,
      "hit10_wilson95": [
        0.01180239266675059,
        0.017855615386723485
      ],
      "recall10": 0.011672057014063776,
      "ndcg10": 0.0050515366567436696
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "all",
      "method": "recent_pop",
      "n": 6060,
      "hits": 127,
      "hit10": 0.020957095709570956,
      "hit10_wilson95": [
        0.017642527525729562,
        0.024878613663677378
      ],
      "recall10": 0.018358351253243792,
      "ndcg10": 0.01022888474401236
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "all",
      "method": "future_control",
      "n": 6060,
      "hits": 94,
      "hit10": 0.015511551155115511,
      "hit10_wilson95": [
        0.01269303447552138,
        0.01894391713732698
      ],
      "recall10": 0.012579148742828816,
      "ndcg10": 0.007606429802916837
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "organic",
      "method": "past_pop",
      "n": 3817,
      "hits": 67,
      "hit10": 0.0175530521351847,
      "hit10_wilson95": [
        0.0138460539062542,
        0.022230150852903488
      ],
      "recall10": 0.014861409179606972,
      "ndcg10": 0.006453458044248303
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "organic",
      "method": "recent_pop",
      "n": 3817,
      "hits": 87,
      "hit10": 0.022792769190463715,
      "hit10_wilson95": [
        0.018516120327257076,
        0.02802898256104362
      ],
      "recall10": 0.02059687588716899,
      "ndcg10": 0.011916049523340183
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "organic",
      "method": "future_control",
      "n": 3817,
      "hits": 71,
      "hit10": 0.018600995546240503,
      "hit10_wilson95": [
        0.01477364023275269,
        0.0233963441522336
      ],
      "recall10": 0.01576883452489982,
      "ndcg10": 0.009342835689568382
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "algorithmic",
      "method": "past_pop",
      "n": 2189,
      "hits": 18,
      "hit10": 0.008222932846048425,
      "hit10_wilson95": [
        0.005207734815333733,
        0.012961138570579352
      ],
      "recall10": 0.005850464443429268,
      "ndcg10": 0.0024365785514255263
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "algorithmic",
      "method": "recent_pop",
      "n": 2189,
      "hits": 35,
      "hit10": 0.015989036089538604,
      "hit10_wilson95": [
        0.011518950488141853,
        0.022154919788100517
      ],
      "recall10": 0.013887619917770672,
      "ndcg10": 0.006701124058950993
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "algorithmic",
      "method": "future_control",
      "n": 2189,
      "hits": 20,
      "hit10": 0.009136592051164915,
      "hit10_wilson95": [
        0.005922327726033437,
        0.014070662941094856
      ],
      "recall10": 0.006779351301964367,
      "ndcg10": 0.0041545000334946985
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "mixed",
      "method": "past_pop",
      "n": 54,
      "hits": 3,
      "hit10": 0.05555555555555555,
      "hit10_wilson95": [
        0.0190732684546909,
        0.15107214318166143
      ],
      "recall10": 0.022222222222222223,
      "ndcg10": 0.01195911733148866
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "mixed",
      "method": "recent_pop",
      "n": 54,
      "hits": 5,
      "hit10": 0.09259259259259259,
      "hit10_wilson95": [
        0.04020113475702944,
        0.19909882590962688
      ],
      "recall10": 0.04135802469135803,
      "ndcg10": 0.03397629542743873
    },
    {
      "fraction": 0.8,
      "cutoff": 20799992,
      "recent_start": 16640009,
      "group": "mixed",
      "method": "future_control",
      "n": 54,
      "hits": 3,
      "hit10": 0.05555555555555555,
      "hit10_wilson95": [
        0.0190732684546909,
        0.15107214318166143
      ],
      "recall10": 0.022222222222222223,
      "ndcg10": 0.02479926306062252
    }
  ],
  "paired": [
    {
      "fraction": 0.6,
      "comparison": "recent_pop minus past_pop",
      "n": 5516,
      "improved": 98,
      "worsened": 45,
      "delta_hit10": 0.009608411892675852,
      "normal95": [
        0.005366471839883784,
        0.01385035194546792
      ]
    },
    {
      "fraction": 0.6,
      "comparison": "future_control minus past_pop",
      "n": 5516,
      "improved": 100,
      "worsened": 26,
      "delta_hit10": 0.013415518491660623,
      "normal95": [
        0.009442333486880986,
        0.017388703496440262
      ]
    },
    {
      "fraction": 0.7,
      "comparison": "recent_pop minus past_pop",
      "n": 5795,
      "improved": 56,
      "worsened": 26,
      "delta_hit10": 0.005176876617773943,
      "normal95": [
        0.002116778374002352,
        0.008236974861545534
      ]
    },
    {
      "fraction": 0.7,
      "comparison": "future_control minus past_pop",
      "n": 5795,
      "improved": 23,
      "worsened": 13,
      "delta_hit10": 0.001725625539257981,
      "normal95": [
        -0.00030339873966470934,
        0.0037546498181806713
      ]
    },
    {
      "fraction": 0.8,
      "comparison": "recent_pop minus past_pop",
      "n": 6060,
      "improved": 66,
      "worsened": 27,
      "delta_hit10": 0.006435643564356435,
      "normal95": [
        0.0033205297114079144,
        0.009550757417304956
      ]
    },
    {
      "fraction": 0.8,
      "comparison": "future_control minus past_pop",
      "n": 6060,
      "improved": 13,
      "worsened": 7,
      "delta_hit10": 0.0009900990099009901,
      "normal95": [
        -0.0004562389082248177,
        0.002436436928026798
      ]
    }
  ]
}