{ "meta": { "title": "Campaign Impact Check", "prepared": "October 2026", "data": "Kevin Hillstrom's MineThatData email test (2008): 64,000 US customers of an online retailer", "window": "two weeks after the email", "currency": "USD", "source_url": "https://blog.minethatdata.com/2008/03/minethatdata-e-mail-analytics-and-data.html", "source_sha256": "bab6578f60db5d792f1c2372c502f029152a5249cf5ea84390f3b7f885d7234f", "label": "Sample study using a public 2008 experiment. Not a Dunwell client. Kevin Hillstrom and MineThatData do not endorse Dunwell." }, "audit": { "source_sha256": "bab6578f60db5d792f1c2372c502f029152a5249cf5ea84390f3b7f885d7234f", "file_checks": { "rows": 64000, "columns": 12, "missing_values": 0, "exact_copy_rows": 6562, "spend_without_purchase": 0, "purchase_without_spend": 0, "purchase_without_visit": 0, "arms": [ "Mens E-Mail", "No E-Mail", "Womens E-Mail" ] }, "allocation": { "counts": { "No E-Mail": 21306, "Mens E-Mail": 21307, "Womens E-Mail": 21387 }, "share": { "No E-Mail": 0.3329, "Mens E-Mail": 0.3329, "Womens E-Mail": 0.3342 }, "chi_square_p_equal_thirds": 0.9037 }, "balance_max_abs_std_diff": 0.0164, "balance_columns_over_0_1": 0, "balance_p_values": { "recency": 0.7941, "history": 0.6669, "history_segment": 0.358, "zip_code": 0.5795, "channel": 0.458, "mens": 0.6717, "womens": 0.7289, "newbie": 0.9339 }, "split_seed": 20261011, "split_counts": { "test": { "Mens E-Mail": 6392, "No E-Mail": 6392, "Womens E-Mail": 6416 }, "train": { "Mens E-Mail": 14915, "No E-Mail": 14914, "Womens E-Mail": 14971 } } }, "balance": [ { "arm": "Mens E-Mail", "column": "recency", "email_mean": 5.7736, "no_email_mean": 5.7497, "std_diff": 0.0068 }, { "arm": "Mens E-Mail", "column": "history", "email_mean": 242.8359, "no_email_mean": 240.8827, "std_diff": 0.0076 }, { "arm": "Mens E-Mail", "column": "mens", "email_mean": 0.5509, "no_email_mean": 0.5532, "std_diff": -0.0046 }, { "arm": "Mens E-Mail", "column": "womens", "email_mean": 0.5514, "no_email_mean": 0.5476, "std_diff": 0.0076 }, { "arm": "Mens E-Mail", "column": "newbie", "email_mean": 0.5015, "no_email_mean": 0.502, "std_diff": -0.0009 }, { "arm": "Mens E-Mail", "column": "history_segment_1) $0 - $100", "email_mean": 0.3625, "no_email_mean": 0.3573, "std_diff": 0.0109 }, { "arm": "Mens E-Mail", "column": "history_segment_2) $100 - $200", "email_mean": 0.2202, "no_email_mean": 0.227, "std_diff": -0.0164 }, { "arm": "Mens E-Mail", "column": "history_segment_3) $200 - $350", "email_mean": 0.192, "no_email_mean": 0.1898, "std_diff": 0.0055 }, { "arm": "Mens E-Mail", "column": "history_segment_4) $350 - $500", "email_mean": 0.0984, "no_email_mean": 0.0997, "std_diff": -0.0043 }, { "arm": "Mens E-Mail", "column": "history_segment_5) $500 - $750", "email_mean": 0.075, "no_email_mean": 0.0775, "std_diff": -0.0097 }, { "arm": "Mens E-Mail", "column": "history_segment_6) $750 - $1,000", "email_mean": 0.0302, "no_email_mean": 0.0292, "std_diff": 0.0061 }, { "arm": "Mens E-Mail", "column": "history_segment_7) $1,000 +", "email_mean": 0.0218, "no_email_mean": 0.0195, "std_diff": 0.0158 }, { "arm": "Mens E-Mail", "column": "zip_code_Rural", "email_mean": 0.1522, "no_email_mean": 0.1473, "std_diff": 0.0137 }, { "arm": "Mens E-Mail", "column": "zip_code_Surburban", "email_mean": 0.4459, "no_email_mean": 0.4518, "std_diff": -0.0117 }, { "arm": "Mens E-Mail", "column": "zip_code_Urban", "email_mean": 0.4019, "no_email_mean": 0.4009, "std_diff": 0.002 }, { "arm": "Mens E-Mail", "column": "channel_Multichannel", "email_mean": 0.1209, "no_email_mean": 0.1223, "std_diff": -0.0042 }, { "arm": "Mens E-Mail", "column": "channel_Phone", "email_mean": 0.4337, "no_email_mean": 0.4378, "std_diff": -0.0083 }, { "arm": "Mens E-Mail", "column": "channel_Web", "email_mean": 0.4454, "no_email_mean": 0.4399, "std_diff": 0.011 }, { "arm": "Womens E-Mail", "column": "recency", "email_mean": 5.7678, "no_email_mean": 5.7497, "std_diff": 0.0052 }, { "arm": "Womens E-Mail", "column": "history", "email_mean": 242.5366, "no_email_mean": 240.8827, "std_diff": 0.0065 }, { "arm": "Womens E-Mail", "column": "mens", "email_mean": 0.5489, "no_email_mean": 0.5532, "std_diff": -0.0086 }, { "arm": "Womens E-Mail", "column": "womens", "email_mean": 0.5501, "no_email_mean": 0.5476, "std_diff": 0.0049 }, { "arm": "Womens E-Mail", "column": "newbie", "email_mean": 0.5032, "no_email_mean": 0.502, "std_diff": 0.0026 }, { "arm": "Womens E-Mail", "column": "history_segment_1) $0 - $100", "email_mean": 0.3569, "no_email_mean": 0.3573, "std_diff": -0.0007 }, { "arm": "Womens E-Mail", "column": "history_segment_2) $100 - $200", "email_mean": 0.221, "no_email_mean": 0.227, "std_diff": -0.0143 }, { "arm": "Womens E-Mail", "column": "history_segment_3) $200 - $350", "email_mean": 0.1943, "no_email_mean": 0.1898, "std_diff": 0.0114 }, { "arm": "Womens E-Mail", "column": "history_segment_4) $350 - $500", "email_mean": 0.1023, "no_email_mean": 0.0997, "std_diff": 0.0087 }, { "arm": "Womens E-Mail", "column": "history_segment_5) $500 - $750", "email_mean": 0.0777, "no_email_mean": 0.0775, "std_diff": 0.0006 }, { "arm": "Womens E-Mail", "column": "history_segment_6) $750 - $1,000", "email_mean": 0.0277, "no_email_mean": 0.0292, "std_diff": -0.0088 }, { "arm": "Womens E-Mail", "column": "history_segment_7) $1,000 +", "email_mean": 0.02, "no_email_mean": 0.0195, "std_diff": 0.0035 }, { "arm": "Womens E-Mail", "column": "zip_code_Rural", "email_mean": 0.1487, "no_email_mean": 0.1473, "std_diff": 0.004 }, { "arm": "Womens E-Mail", "column": "zip_code_Surburban", "email_mean": 0.4512, "no_email_mean": 0.4518, "std_diff": -0.0011 }, { "arm": "Womens E-Mail", "column": "zip_code_Urban", "email_mean": 0.4001, "no_email_mean": 0.4009, "std_diff": -0.0018 }, { "arm": "Womens E-Mail", "column": "channel_Multichannel", "email_mean": 0.1206, "no_email_mean": 0.1223, "std_diff": -0.0053 }, { "arm": "Womens E-Mail", "column": "channel_Phone", "email_mean": 0.442, "no_email_mean": 0.4378, "std_diff": 0.0086 }, { "arm": "Womens E-Mail", "column": "channel_Web", "email_mean": 0.4374, "no_email_mean": 0.4399, "std_diff": -0.0051 } ], "observed": { "arms": { "none": { "customers": 21306, "visitors": 2262, "buyers": 122, "visit_rate": 0.10616727682343002, "purchase_rate": 0.005726086548390125, "spend_per_customer": 0.6527893551112361, "spend_total": 13908.329999999998, "spend_per_buyer": 114.00270491803279, "buyers_per_visitor": 0.053934571175950484 }, "mens": { "customers": 21307, "visitors": 3894, "buyers": 267, "visit_rate": 0.18275684047496127, "purchase_rate": 0.01253109306800582, "spend_per_customer": 1.422616511005773, "spend_total": 30311.690000000002, "spend_per_buyer": 113.52692883895132, "buyers_per_visitor": 0.06856702619414484 }, "womens": { "customers": 21387, "visitors": 3238, "buyers": 189, "visit_rate": 0.151400383410483, "purchase_rate": 0.008837144059475383, "spend_per_customer": 1.0772015710478329, "spend_total": 23038.11, "spend_per_buyer": 121.8947619047619, "buyers_per_visitor": 0.05836936380481779 } }, "effects": { "mens": { "visit": { "diff": 0.07658956365153125, "ci": [ 0.0699533792766632, 0.08322574802639929 ], "p": 5.685165482802317e-112, "relative": 0.721404616781399 }, "conversion": { "diff": 0.006805006519615695, "ci": [ 0.005000138215573684, 0.008609874823657705 ], "p": 1.5232235005955427e-13, "relative": 1.188421876286328, "p_holm": 3.0464470011910854e-13 }, "spend": { "diff": 0.7698271558945368, "ci": [ 0.48728220297646707, 1.0550273280388995 ], "p": 1.163814968225482e-07 }, "extra_buyers_per_1000": 6.805006519615695, "extra_buyers_per_1000_ci": [ 5.000138215573684, 8.609874823657705 ], "emails_per_extra_buyer": 146.95063070365345, "extra_spend_per_1000": 769.8271558945368, "extra_spend_per_1000_ci": [ 487.2822029764671, 1055.0273280388994 ], "share_of_buyers_who_would_have_bought_anyway": 0.45695028496834605 }, "womens": { "visit": { "diff": 0.045233106587052985, "ci": [ 0.038893709324730276, 0.051572503849375693 ], "p": 3.1824031593349214e-44, "relative": 0.4260550702669102 }, "conversion": { "diff": 0.003111057511085258, "ci": [ 0.0014986481930026369, 0.004723466829167879 ], "p": 0.00015710512910668448, "relative": 0.5433130436982172, "p_holm": 0.00015710512910668448 }, "spend": { "diff": 0.4244122159365967, "ci": [ 0.16497313614282347, 0.6782294408535456 ], "p": 0.0011293971023632495 }, "extra_buyers_per_1000": 3.111057511085258, "extra_buyers_per_1000_ci": [ 1.498648193002637, 4.723466829167879 ], "emails_per_extra_buyer": 321.4341092817539, "extra_spend_per_1000": 424.41221593659674, "extra_spend_per_1000_ci": [ 164.97313614282348, 678.2294408535456 ], "share_of_buyers_who_would_have_bought_anyway": 0.6479566825948126 } }, "mens_vs_womens": { "conversion_diff": 0.003693949008530437, "ci": [ 0.0017434768970844394, 0.005644421119976434 ], "spend_diff": 0.34541493995794004, "spend_ci": [ 0.03214538497303764, 0.649919975886102 ] }, "buyer_spend": { "none": { "median": 88.36500000000001, "mean": 114.00270491803279 }, "mens": { "median": 66.98, "mean": 113.52692883895132 }, "womens": { "median": 94.71, "mean": 121.8947619047619 } }, "attributed_vs_real": { "mens": 1.8414520297496348, "womens": 2.8405595293520123 }, "extra_visits_per_extra_buyer": { "mens": 11.254884683910127, "womens": 14.53946332585601 }, "by_bought": { "mens": [ { "group": "Both", "customers": 6448, "share": 0.10075, "base_rate": 0.009306654257794323, "email_rate": 0.024300779458963778, "effect": 0.014994125201169455, "se": 0.003893797863329508, "ci": [ 0.0073622813890436185, 0.02262596901329529 ], "spend_effect": 1.824986420861755, "spend_ci": [ 0.4901365617234146, 3.1598362800000954 ] }, { "group": "Men's only", "customers": 28818, "share": 0.45028125, "base_rate": 0.005499066196306288, "email_rate": 0.011508683825068006, "effect": 0.006009617628761718, "se": 0.0013257658215198949, "ci": [ 0.003411116618582724, 0.008608118638940713 ], "spend_effect": 0.682735312299813, "spend_ci": [ 0.24716406172448274, 1.1183065628751432 ] }, { "group": "Women's only", "customers": 28734, "share": 0.44896875, "base_rate": 0.005147599537766572, "email_rate": 0.010869565217391304, "effect": 0.005721965679624732, "se": 0.0012890587013849987, "ci": [ 0.0031954106249101346, 0.00824852073433933 ], "spend_effect": 0.6157308065834535, "spend_ci": [ 0.2696714823874825, 0.9617901307794244 ] } ], "womens": [ { "group": "Both", "customers": 6448, "share": 0.10075, "base_rate": 0.009306654257794323, "email_rate": 0.014636449480642116, "effect": 0.005329795222847792, "se": 0.0033316281492257356, "ci": [ -0.001200195949634649, 0.011859786395330234 ], "spend_effect": 0.16188124260971226, "spend_ci": [ -0.8133551647202163, 1.1371176499396407 ] }, { "group": "Men's only", "customers": 28818, "share": 0.45028125, "base_rate": 0.005499066196306288, "email_rate": 0.006131781334441904, "effect": 0.0006327151381356165, "se": 0.0010958022683162138, "ci": [ -0.0015150573077641625, 0.0027804875840353955 ], "spend_effect": 0.27835448170778665, "spend_ci": [ -0.12356921545452715, 0.6802781788701004 ] }, { "group": "Women's only", "customers": 28734, "share": 0.44896875, "base_rate": 0.005147599537766572, "email_rate": 0.010262257696693273, "effect": 0.005114658158926701, "se": 0.001261288117236602, "ci": [ 0.002642533449142961, 0.007586782868710441 ], "spend_effect": 0.6316479517709466, "spend_ci": [ 0.2951273373598922, 0.968168566182001 ] } ] }, "by_group": { "mens|bought": [ { "group": "Both", "customers": 6448, "share": 0.10075, "base_rate": 0.009306654257794323, "email_rate": 0.024300779458963778, "effect": 0.014994125201169455, "se": 0.003893797863329508, "ci": [ 0.0073622813890436185, 0.02262596901329529 ], "spend_effect": 1.824986420861755, "spend_ci": [ 0.4901365617234146, 3.1598362800000954 ] }, { "group": "Men's only", "customers": 28818, "share": 0.45028125, "base_rate": 0.005499066196306288, "email_rate": 0.011508683825068006, "effect": 0.006009617628761718, "se": 0.0013257658215198949, "ci": [ 0.003411116618582724, 0.008608118638940713 ], "spend_effect": 0.682735312299813, "spend_ci": [ 0.24716406172448274, 1.1183065628751432 ] }, { "group": "Women's only", "customers": 28734, "share": 0.44896875, "base_rate": 0.005147599537766572, "email_rate": 0.010869565217391304, "effect": 0.005721965679624732, "se": 0.0012890587013849987, "ci": [ 0.0031954106249101346, 0.00824852073433933 ], "spend_effect": 0.6157308065834535, "spend_ci": [ 0.2696714823874825, 0.9617901307794244 ] } ], "mens|recency_band": [ { "group": "1-3 months", "customers": 22393, "share": 0.349890625, "base_rate": 0.009007797795106211, "email_rate": 0.01633418128263489, "effect": 0.007326383487528679, "se": 0.001830669438712448, "ci": [ 0.0037382713876522814, 0.010914495587405077 ], "spend_effect": 1.047598062490867, "spend_ci": [ 0.4641501936955854, 1.6310459312861485 ] }, { "group": "4-6 months", "customers": 14192, "share": 0.22175, "base_rate": 0.005455308434746118, "email_rate": 0.010485769313075112, "effect": 0.005030460878328993, "se": 0.0018326926287396158, "ci": [ 0.0014383833259993466, 0.00862253843065864 ], "spend_effect": 0.48838743501902304, "spend_ci": [ 0.008860124253868007, 0.9679147457841781 ] }, { "group": "7-9 months", "customers": 14014, "share": 0.21896875, "base_rate": 0.003404979782932539, "email_rate": 0.011305460750853242, "effect": 0.007900480967920703, "se": 0.0017625155547533896, "ci": [ 0.00444595048060406, 0.011355011455237346 ], "spend_effect": 0.6050864572884942, "spend_ci": [ 0.0641924526931773, 1.145980461883811 ] }, { "group": "10-12 months", "customers": 13401, "share": 0.209390625, "base_rate": 0.0029525323642970703, "email_rate": 0.009604645968282331, "effect": 0.0066521136039852605, "se": 0.001671322421894334, "ci": [ 0.003376321657072366, 0.009927905550898154 ], "spend_effect": 0.7720818074828771, "spend_ci": [ 0.2094493937774662, 1.334714221188288 ] } ], "mens|spend_band": [ { "group": "Under $100", "customers": 22970, "share": 0.35890625, "base_rate": 0.004203888596952181, "email_rate": 0.009192128430864838, "effect": 0.004988239833912657, "se": 0.0013149463205837585, "ci": [ 0.0024109450455684902, 0.007565534622256824 ], "spend_effect": 0.5382830109889453, "spend_ci": [ 0.11286738457874923, 0.9636986373991413 ] }, { "group": "$100-200", "customers": 14254, "share": 0.22271875, "base_rate": 0.0041356492969396195, "email_rate": 0.008953314858239182, "effect": 0.004817665561299562, "se": 0.001656251792677637, "ci": [ 0.0015714120476513936, 0.008063919074947731 ], "spend_effect": 0.7042485117040373, "spend_ci": [ 0.19779429484815458, 1.2107027285599201 ] }, { "group": "$200-500", "customers": 18698, "share": 0.29215625, "base_rate": 0.008430609597924773, "email_rate": 0.016971068369161145, "effect": 0.008540458771236372, "se": 0.0020129023159253176, "ci": [ 0.00459517023202275, 0.012485747310449994 ], "spend_effect": 0.8860657722973382, "spend_ci": [ 0.2716247446429436, 1.5005067999517328 ] }, { "group": "$500+", "customers": 8078, "share": 0.12621875, "base_rate": 0.006691449814126394, "email_rate": 0.01811460258780037, "effect": 0.011423152773673978, "se": 0.003007704297937469, "ci": [ 0.005528052349716539, 0.017318253197631417 ], "spend_effect": 1.2694752248692702, "spend_ci": [ 0.3656872711636997, 2.1732631785748406 ] } ], "mens|newbie": [ { "group": "0", "customers": 31856, "share": 0.49775, "base_rate": 0.007727829610781264, "email_rate": 0.013934657753507202, "effect": 0.006206828142725938, "se": 0.001419987915667753, "ci": [ 0.0034236518280171425, 0.008990004457434734 ], "spend_effect": 0.5779669050857943, "spend_ci": [ 0.14158567091293356, 1.014348139258655 ] }, { "group": "1", "customers": 32144, "share": 0.50225, "base_rate": 0.0037400654511453952, "email_rate": 0.011136065880591428, "effect": 0.007396000429446033, "se": 0.0011742685304044174, "ci": [ 0.005094434109853375, 0.009697566749038692 ], "spend_effect": 0.9600198419292101, "spend_ci": [ 0.5937908661124862, 1.326248817745934 ] } ], "mens|channel": [ { "group": "Multichannel", "customers": 7762, "share": 0.12128125, "base_rate": 0.006907137375287797, "email_rate": 0.017074117190531625, "effect": 0.010166979815243827, "se": 0.0030240073996066976, "ci": [ 0.0042399253120147, 0.016094034318472953 ], "spend_effect": 1.2091294052619086, "spend_ci": [ 0.39105217708740825, 2.0272066334364087 ] }, { "group": "Phone", "customers": 28021, "share": 0.437828125, "base_rate": 0.005360780529645117, "email_rate": 0.010822510822510822, "effect": 0.005461730292865705, "se": 0.0013153971072770553, "ci": [ 0.002883551962602677, 0.008039908623128734 ], "spend_effect": 0.5640743919691333, "spend_ci": [ 0.16115329289467256, 0.9669954910435941 ] }, { "group": "Web", "customers": 28217, "share": 0.440890625, "base_rate": 0.005761229062199936, "email_rate": 0.012961011591148578, "effect": 0.007199782528948642, "se": 0.0013997050054511137, "ci": [ 0.0044563607182644596, 0.009943204339632824 ], "spend_effect": 0.8504486670398359, "spend_ci": [ 0.3971820877129374, 1.3037152463667345 ] } ], "mens|zip_code": [ { "group": "Rural", "customers": 9563, "share": 0.149421875, "base_rate": 0.00828289264096846, "email_rate": 0.013567684242984891, "effect": 0.00528479160201643, "se": 0.002596876650801305, "ci": [ 0.00019491336644587246, 0.010374669837586989 ], "spend_effect": 0.7555004171505917, "spend_ci": [ 0.05092197184209013, 1.4600788624590932 ] }, { "group": "Suburban", "customers": 28776, "share": 0.449625, "base_rate": 0.004987012987012987, "email_rate": 0.012419745289969477, "effect": 0.00743273230295649, "se": 0.0013440670223332258, "ci": [ 0.004798360939183367, 0.010067103666729613 ], "spend_effect": 0.8038481020589765, "spend_ci": [ 0.35414812953189573, 1.2535480745860572 ] }, { "group": "Urban", "customers": 25661, "share": 0.400953125, "base_rate": 0.005619292905642706, "email_rate": 0.012262057690061895, "effect": 0.006642764784419188, "se": 0.0014382519344837935, "ci": [ 0.003823790992830953, 0.009461738576007423 ], "spend_effect": 0.7365068185112378, "spend_ci": [ 0.31098246414892106, 1.1620311728735544 ] } ], "womens|bought": [ { "group": "Both", "customers": 6448, "share": 0.10075, "base_rate": 0.009306654257794323, "email_rate": 0.014636449480642116, "effect": 0.005329795222847792, "se": 0.0033316281492257356, "ci": [ -0.001200195949634649, 0.011859786395330234 ], "spend_effect": 0.16188124260971226, "spend_ci": [ -0.8133551647202163, 1.1371176499396407 ] }, { "group": "Men's only", "customers": 28818, "share": 0.45028125, "base_rate": 0.005499066196306288, "email_rate": 0.006131781334441904, "effect": 0.0006327151381356165, "se": 0.0010958022683162138, "ci": [ -0.0015150573077641625, 0.0027804875840353955 ], "spend_effect": 0.27835448170778665, "spend_ci": [ -0.12356921545452715, 0.6802781788701004 ] }, { "group": "Women's only", "customers": 28734, "share": 0.44896875, "base_rate": 0.005147599537766572, "email_rate": 0.010262257696693273, "effect": 0.005114658158926701, "se": 0.001261288117236602, "ci": [ 0.002642533449142961, 0.007586782868710441 ], "spend_effect": 0.6316479517709466, "spend_ci": [ 0.2951273373598922, 0.968168566182001 ] } ], "womens|recency_band": [ { "group": "1-3 months", "customers": 22393, "share": 0.349890625, "base_rate": 0.009007797795106211, "email_rate": 0.011755276516163505, "effect": 0.002747478721057293, "se": 0.001658910564897147, "ci": [ -0.0005039859861411151, 0.005998943428255701 ], "spend_effect": 0.3585999956035171, "spend_ci": [ -0.11543762884398845, 0.8326376200510226 ] }, { "group": "4-6 months", "customers": 14192, "share": 0.22175, "base_rate": 0.005455308434746118, "email_rate": 0.007994950557542605, "effect": 0.0025396421227964865, "se": 0.0016754172633283544, "ci": [ -0.0007441757133270881, 0.005823459958920061 ], "spend_effect": 0.6654827359516472, "spend_ci": [ 0.10983381600974595, 1.2211316558935486 ] }, { "group": "7-9 months", "customers": 14014, "share": 0.21896875, "base_rate": 0.003404979782932539, "email_rate": 0.006699805489518046, "effect": 0.0032948257065855075, "se": 0.0014698400931644668, "ci": [ 0.0004139391239831526, 0.006175712289187863 ], "spend_effect": 0.29948940895002096, "spend_ci": [ -0.24847915001236676, 0.8474579679124087 ] }, { "group": "10-12 months", "customers": 13401, "share": 0.209390625, "base_rate": 0.0029525323642970703, "email_rate": 0.007078080070780801, "effect": 0.004125547706483731, "se": 0.0014910104246412348, "ci": [ 0.0012031672741869108, 0.0070479281387805515 ], "spend_effect": 0.41000612680732895, "spend_ci": [ -0.021799236140184086, 0.841811489754842 ] } ], "womens|spend_band": [ { "group": "Under $100", "customers": 22970, "share": 0.35890625, "base_rate": 0.004203888596952181, "email_rate": 0.007990568509300498, "effect": 0.003786679912348317, "se": 0.0012602752490624188, "ci": [ 0.001316540424185976, 0.006256819400510658 ], "spend_effect": 0.6258197999215559, "spend_ci": [ 0.18813180803538782, 1.063507791807724 ] }, { "group": "$100-200", "customers": 14254, "share": 0.22271875, "base_rate": 0.0041356492969396195, "email_rate": 0.007827374656230167, "effect": 0.0036917253592905477, "se": 0.0015794201827654122, "ci": [ 0.00059606180107034, 0.006787388917510755 ], "spend_effect": 0.4583933890504246, "spend_ci": [ 0.04388866122525237, 0.8728981168755968 ] }, { "group": "$200-500", "customers": 18698, "share": 0.29215625, "base_rate": 0.008430609597924773, "email_rate": 0.007252088916916285, "effect": -0.0011785206810084872, "se": 0.0015780802476928443, "ci": [ -0.004271557966486462, 0.0019145166044694873 ], "spend_effect": -0.24643035164636062, "spend_ci": [ -0.7028460590429662, 0.20998535575024485 ] }, { "group": "$500+", "customers": 8078, "share": 0.12621875, "base_rate": 0.006691449814126394, "email_rate": 0.016772269847185987, "effect": 0.010080820033059594, "se": 0.0029355316147988016, "ci": [ 0.004327178068053943, 0.015834461998065246 ], "spend_effect": 1.349456245921242, "spend_ci": [ 0.3909381339723721, 2.307974357870112 ] } ], "womens|newbie": [ { "group": "0", "customers": 31856, "share": 0.49775, "base_rate": 0.007727829610781264, "email_rate": 0.008659638554216868, "effect": 0.0009318089434356032, "se": 0.0012372145074927006, "ci": [ -0.00149313149125009, 0.0033567493781212964 ], "spend_effect": 0.11395712053756446, "spend_ci": [ -0.2754754786063253, 0.5033897196814543 ] }, { "group": "1", "customers": 32144, "share": 0.50225, "base_rate": 0.0037400654511453952, "email_rate": 0.009012357149493635, "effect": 0.00527229169834824, "se": 0.0010854464828323929, "ci": [ 0.0031448165919967497, 0.007399766804699729 ], "spend_effect": 0.7322877956561398, "spend_ci": [ 0.40119314289962377, 1.0633824484126557 ] } ], "womens|channel": [ { "group": "Multichannel", "customers": 7762, "share": 0.12128125, "base_rate": 0.006907137375287797, "email_rate": 0.013958898797983715, "effect": 0.0070517614226959175, "se": 0.0028229662627743514, "ci": [ 0.0015187475476581887, 0.012584775297733646 ], "spend_effect": 1.1575358651270653, "spend_ci": [ 0.2588654032549704, 2.05620632699916 ] }, { "group": "Phone", "customers": 28021, "share": 0.437828125, "base_rate": 0.005360780529645117, "email_rate": 0.00708694732388407, "effect": 0.0017261667942389536, "se": 0.001147166015076057, "ci": [ -0.0005222785953101182, 0.003974612183788025 ], "spend_effect": 0.23247350450950877, "spend_ci": [ -0.11920642559875161, 0.5841534346177691 ] }, { "group": "Web", "customers": 28217, "share": 0.440890625, "base_rate": 0.005761229062199936, "email_rate": 0.009193927731451786, "effect": 0.0034326986692518498, "se": 0.0012589565979448962, "ci": [ 0.0009651437372798534, 0.005900253601223846 ], "spend_effect": 0.4163166914707608, "spend_ci": [ 0.024553976811312994, 0.8080794061302086 ] } ], "womens|zip_code": [ { "group": "Rural", "customers": 9563, "share": 0.149421875, "base_rate": 0.00828289264096846, "email_rate": 0.011631562401760453, "effect": 0.003348669760791992, "se": 0.0024961752012155687, "ci": [ -0.0015438336335905226, 0.008241173155174506 ], "spend_effect": 0.5148413350253109, "spend_ci": [ -0.11955397717856298, 1.1492366472291848 ] }, { "group": "Suburban", "customers": 28776, "share": 0.449625, "base_rate": 0.004987012987012987, "email_rate": 0.0075647668393782385, "effect": 0.0025777538523652516, "se": 0.0011373349683999445, "ci": [ 0.00034857731430136063, 0.004806930390429143 ], "spend_effect": 0.1771713935805127, "spend_ci": [ -0.19359375570063053, 0.5479365428616559 ] }, { "group": "Urban", "customers": 25661, "share": 0.400953125, "base_rate": 0.005619292905642706, "email_rate": 0.009233286582515194, "effect": 0.003613993676872488, "se": 0.0013127609160034875, "ci": [ 0.0010409822815056523, 0.006187005072239324 ], "spend_effect": 0.669135334806898, "spend_ci": [ 0.24825625908277998, 1.090014410531016 ] } ] }, "segment_checks": [ { "arm": "womens", "variable": "spend_band", "label": "Spend last year", "train_p": 0.03258043803355107, "test_p": 0.044474002463064814, "same_direction_on_test": true, "confirmed": true, "high_group": "$500+", "low_group": "$200-500", "groups": [ { "group": "Under $100", "customers": 22970, "share": 0.35890625, "base_rate": 0.004203888596952181, "email_rate": 0.007990568509300498, "effect": 0.003786679912348317, "se": 0.0012602752490624188, "ci": [ 0.001316540424185976, 0.006256819400510658 ], "spend_effect": 0.6258197999215559, "spend_ci": [ 0.18813180803538782, 1.063507791807724 ] }, { "group": "$100-200", "customers": 14254, "share": 0.22271875, "base_rate": 0.0041356492969396195, "email_rate": 0.007827374656230167, "effect": 0.0036917253592905477, "se": 0.0015794201827654122, "ci": [ 0.00059606180107034, 0.006787388917510755 ], "spend_effect": 0.4583933890504246, "spend_ci": [ 0.04388866122525237, 0.8728981168755968 ] }, { "group": "$200-500", "customers": 18698, "share": 0.29215625, "base_rate": 0.008430609597924773, "email_rate": 0.007252088916916285, "effect": -0.0011785206810084872, "se": 0.0015780802476928443, "ci": [ -0.004271557966486462, 0.0019145166044694873 ], "spend_effect": -0.24643035164636062, "spend_ci": [ -0.7028460590429662, 0.20998535575024485 ] }, { "group": "$500+", "customers": 8078, "share": 0.12621875, "base_rate": 0.006691449814126394, "email_rate": 0.016772269847185987, "effect": 0.010080820033059594, "se": 0.0029355316147988016, "ci": [ 0.004327178068053943, 0.015834461998065246 ], "spend_effect": 1.349456245921242, "spend_ci": [ 0.3909381339723721, 2.307974357870112 ] } ] }, { "arm": "womens", "variable": "newbie", "label": "New in the last 12 months", "train_p": 0.01261343046380769, "test_p": 0.32519933045915445, "same_direction_on_test": true, "confirmed": false, "high_group": "1", "low_group": "0", "groups": [ { "group": "0", "customers": 31856, "share": 0.49775, "base_rate": 0.007727829610781264, "email_rate": 0.008659638554216868, "effect": 0.0009318089434356032, "se": 0.0012372145074927006, "ci": [ -0.00149313149125009, 0.0033567493781212964 ], "spend_effect": 0.11395712053756446, "spend_ci": [ -0.2754754786063253, 0.5033897196814543 ] }, { "group": "1", "customers": 32144, "share": 0.50225, "base_rate": 0.0037400654511453952, "email_rate": 0.009012357149493635, "effect": 0.00527229169834824, "se": 0.0010854464828323929, "ci": [ 0.0031448165919967497, 0.007399766804699729 ], "spend_effect": 0.7322877956561398, "spend_ci": [ 0.40119314289962377, 1.0633824484126557 ] } ] } ], "segments_not_flagged": [ { "arm": "mens", "variable": "bought", "label": "What they bought last year", "train_p": 0.15359246577590654 }, { "arm": "mens", "variable": "recency_band", "label": "Months since last order", "train_p": 0.5587364918645774 }, { "arm": "mens", "variable": "spend_band", "label": "Spend last year", "train_p": 0.44647185156436764 }, { "arm": "mens", "variable": "newbie", "label": "New in the last 12 months", "train_p": 0.5229112977472834 }, { "arm": "mens", "variable": "channel", "label": "Where they bought", "train_p": 0.9025297677537713 }, { "arm": "mens", "variable": "zip_code", "label": "Area", "train_p": 0.41126865893415876 }, { "arm": "womens", "variable": "bought", "label": "What they bought last year", "train_p": 0.17904793894359666 }, { "arm": "womens", "variable": "recency_band", "label": "Months since last order", "train_p": 0.6673729487578379 }, { "arm": "womens", "variable": "channel", "label": "Where they bought", "train_p": 0.45059898796882236 }, { "arm": "womens", "variable": "zip_code", "label": "Area", "train_p": 0.9622180459869036 } ] }, "targeting": { "chosen": "dr_learner", "chosen_name": "DR-learner", "frozen_at_utc": "2026-10-11T12:07:48Z", "rerun_at_utc": "2026-10-11T12:26:48Z", "names": { "logistic": "Logistic regression", "t_learner": "T-learner", "x_learner": "X-learner", "dr_learner": "DR-learner", "causal_forest": "Causal forest", "tarnet": "Neural network", "recent_spenders": "Simple rule: recent buyers first", "likely_buyers": "Most likely to buy (not uplift)", "random_mens": "Random customers, men's email" }, "models": [ "dr_learner", "causal_forest", "x_learner", "t_learner", "logistic", "tarnet" ], "cv_buyers_per_1000": { "logistic": 7.966402751912094, "t_learner": 8.310421062628398, "x_learner": 8.422207939555266, "dr_learner": 8.992237655247521, "causal_forest": 8.7915509054004, "tarnet": 7.699422530422983, "random_mens": 8.244659997461815 }, "cv_rank": [ "dr_learner", "causal_forest", "x_learner", "t_learner", "logistic", "tarnet" ], "test_rank": [ "tarnet", "t_learner", "logistic", "x_learner", "causal_forest", "dr_learner" ], "test_customers": 19200, "test_buyers": 191, "overall_vs_random_per_1000": { "dr_learner": { "diff": 0.27887103268112623, "ci": [ -0.9855029609752819, 1.5985111764166495 ], "prob_better": 0.6675 }, "causal_forest": { "diff": 0.5917621465734919, "ci": [ -0.7862658152589574, 2.079970686463638 ], "prob_better": 0.799 }, "x_learner": { "diff": 0.7988932029542338, "ci": [ -0.5298793490510434, 2.2422583773732154 ], "prob_better": 0.872 }, "t_learner": { "diff": 1.2153986123552203, "ci": [ -0.2367613182286732, 2.7351679396815594 ], "prob_better": 0.9475 }, "logistic": { "diff": 1.0072922096094772, "ci": [ -0.5436207662569359, 2.536663046339525 ], "prob_better": 0.8985 }, "tarnet": { "diff": 1.272423863162287, "ci": [ -0.06016635743435471, 2.602276386460289 ], "prob_better": 0.97 }, "recent_spenders": { "diff": 0.646641635377556, "ci": [ -0.6623513138038106, 1.9582419190555806 ], "prob_better": 0.8175 }, "likely_buyers": { "diff": 1.1420525657071345, "ci": [ -0.24001370002224576, 2.5735946779604997 ], "prob_better": 0.943 } }, "best_on_test": "tarnet", "any_rule_clearly_better": false, "k_curve": [ 0.0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1.0 ], "curves": { "conversion": { "dr_learner": { "ipw": [ 0.004380475594493116, 0.004691611084928479, 0.00531739331271321, 0.006099035889625124, 0.006411927003517489, 0.007194154788248402, 0.008132828129925499, 0.008915055914656413, 0.008915055914656413, 0.009071501471602595, 0.009226776612910777, 0.009695528075930323, 0.010321310303715055, 0.010321310303715055, 0.010945922115861784, 0.01125881322975415, 0.0117281499005927, 0.0117281499005927, 0.012510377685323613, 0.01297971435616216, 0.01297971435616216 ], "dr": [ 0.004405069478270785, 0.00474142041707018, 0.005149857190940809, 0.005917761918898903, 0.006205865726678005, 0.006983474697787259, 0.007967904637223416, 0.008690354738901321, 0.008672173366053979, 0.00885812821588325, 0.008995909363574124, 0.009509685655602845, 0.01017873348459452, 0.010166064272953893, 0.010798750662518047, 0.011076981702635819, 0.011577923926264158, 0.011623349994045578, 0.012401670700402661, 0.012816768037674505, 0.012803373684152675 ] }, "causal_forest": { "ipw": [ 0.004380475594493116, 0.005474424077478394, 0.0056302844266055765, 0.006256066654390307, 0.00703829443912122, 0.007507045902140769, 0.008445719243817864, 0.008758025149891229, 0.008758025149891229, 0.009227361820729776, 0.009540252934622143, 0.01063420141760742, 0.010790061766734601, 0.011570533928008515, 0.012351006089282425, 0.01266389720317479, 0.013601985337032888, 0.013601985337032888, 0.013445539780086703, 0.013914876450925255, 0.014227767564817619 ], "dr": [ 0.004405069478270785, 0.005402915211218435, 0.005602797880820658, 0.006097576012518309, 0.0068828731702371, 0.007281606138674386, 0.008224877858240448, 0.00854154343737564, 0.008554053727797339, 0.008972145686569443, 0.009285756950778159, 0.010433577450764306, 0.01058613103969128, 0.011332773168654223, 0.012086123504250729, 0.01241345762628404, 0.013408619718564446, 0.013402318466755707, 0.013242442951844553, 0.013715141566514651, 0.014008527812727113 ] }, "x_learner": { "ipw": [ 0.004380475594493116, 0.004535750735801297, 0.005473838869659393, 0.005943175540497941, 0.006881848882175038, 0.007819351808214131, 0.009069745848145594, 0.009069160640326594, 0.009538497311165139, 0.009694942868111322, 0.010006663566365688, 0.01016310912331187, 0.010474829821566235, 0.010630690170693418, 0.011100026841531966, 0.011724053445859695, 0.012348665258006424, 0.01281800192884497, 0.013443784156629703, 0.013287338599683519, 0.013756090062703069 ], "dr": [ 0.004405069478270785, 0.004485326711620636, 0.005295190707078858, 0.005693090884848956, 0.006672708410432364, 0.007615215407161674, 0.008843658980379988, 0.008800230187875712, 0.009291411966336656, 0.009437146849603117, 0.009716166033826036, 0.009903055789393602, 0.010163664575161274, 0.010315966123351701, 0.010802645511231375, 0.011413998632954267, 0.01205456497035931, 0.012570356151742264, 0.013212953860530226, 0.01304061002017521, 0.013500148449098774 ] }, "t_learner": { "ipw": [ 0.004380475594493116, 0.005004502198820845, 0.0056302844266055765, 0.006099621097444124, 0.00766232104344895, 0.008444548828179863, 0.009069745848145594, 0.009225606197272777, 0.010007248774184688, 0.010475415029385236, 0.0107877209354586, 0.010474829821566235, 0.0107871357276396, 0.010942996076766782, 0.011254716775021146, 0.011724053445859695, 0.011724053445859695, 0.012349250465825424, 0.012818587136663973, 0.013757260478341067, 0.01328792380750252 ], "dr": [ 0.004405069478270785, 0.004780903298730588, 0.005359135297708859, 0.005819444073834202, 0.007360638628691767, 0.00812696118113187, 0.00872366364486256, 0.008911889029403722, 0.009656544038784057, 0.010154105252952385, 0.010470238650520809, 0.010178107916094725, 0.010465927837191566, 0.010647382332001968, 0.011002804110674497, 0.011462450052529919, 0.011453674040331, 0.012068050782152718, 0.01259957838726572, 0.013516080523969475, 0.0130477276087321 ] }, "logistic": { "ipw": [ 0.004380475594493116, 0.005162703379224029, 0.005786144775732758, 0.007348844721737582, 0.007818181392576132, 0.007818181392576132, 0.007974626949522315, 0.008756269526434227, 0.009069160640326594, 0.010164279538949873, 0.010789476558915602, 0.010789476558915602, 0.01125822802193515, 0.01188342504190088, 0.013290849846597525, 0.013446124987905705, 0.013759016101798072, 0.014227767564817619, 0.014695933820018166, 0.015008824933910531, 0.015008824933910531 ], "dr": [ 0.004405069478270785, 0.005000987746273345, 0.00552416270860561, 0.007157087707915038, 0.007562099811972513, 0.007509698773870492, 0.007700192159301452, 0.00854031214286308, 0.008803630004261497, 0.009888036544642542, 0.010547880511209378, 0.010548170214207448, 0.011033144099305276, 0.011631360046528256, 0.013030338088920068, 0.013220345778699011, 0.013566980521305888, 0.014053687765270485, 0.01452909899391918, 0.014835505238447866, 0.014870687264707732 ] }, "tarnet": { "ipw": [ 0.004380475594493116, 0.005475594493116395, 0.005944931163954944, 0.0071964956195244055, 0.0076658322903629525, 0.008135168961201502, 0.008291614518147684, 0.009230287859824781, 0.00969962453066333, 0.010481852315394242, 0.01095118898623279, 0.011420525657071337, 0.011733416770963704, 0.012359198998748435, 0.012828535669586983, 0.013297872340425532, 0.013454317897371712, 0.013767209011264079, 0.013767209011264079, 0.014392991239048811, 0.014080100125156446 ], "dr": [ 0.004405069478270785, 0.005365843363237361, 0.005757472722015442, 0.006923287582351681, 0.00741959351691313, 0.007819165609072384, 0.007979915896522775, 0.008975614124974876, 0.009463375650724005, 0.010271225603841581, 0.01078148055942402, 0.011242930913846715, 0.011562279351377868, 0.012167901419923527, 0.012617251893746084, 0.013062297517957275, 0.013247262389974906, 0.013576782998545586, 0.01357935467686224, 0.014181346690094519, 0.013918157190860262 ] }, "recent_spenders": { "ipw": [ 0.004380475594493116, 0.004849812265331665, 0.00578848560700876, 0.006727158948685856, 0.007509386733416771, 0.007509386733416771, 0.007978723404255319, 0.008291614518147684, 0.009230287859824781, 0.009543178973717146, 0.009386733416770963, 0.010638297872340425, 0.010638297872340425, 0.01095118898623279, 0.011420525657071337, 0.012359198998748435, 0.012672090112640802, 0.01329787234042553, 0.013767209011264079, 0.013923654568210262, 0.014080100125156446 ], "dr": [ 0.004405069478270785, 0.004890372696284479, 0.005889497591470643, 0.006818034509150953, 0.007449673798985704, 0.0074809589110894055, 0.007901769041964012, 0.008249173554934337, 0.009128136323653488, 0.009403585981527067, 0.009240375429333803, 0.010463874642500067, 0.010532841166369874, 0.010803109090965938, 0.011263320464464184, 0.01221311981097677, 0.01253220580803202, 0.013206754081387231, 0.013633734169280059, 0.013767575218956954, 0.013918157190860262 ] }, "likely_buyers": { "ipw": [ 0.004380475594493116, 0.005632040050062578, 0.006727158948685856, 0.007509386733416771, 0.008291614518147684, 0.00876095118898623, 0.009073842302878598, 0.009386733416770963, 0.00969962453066333, 0.009699624530663328, 0.00969962453066333, 0.010012515644555693, 0.010012515644555693, 0.010794743429286609, 0.01095118898623279, 0.01095118898623279, 0.011889862327909888, 0.012984981226533167, 0.013297872340425532, 0.01329787234042553, 0.014080100125156446 ], "dr": [ 0.004405069478270785, 0.005422526307570636, 0.006520300682861268, 0.007392882459560325, 0.00811825928932483, 0.008535181224361796, 0.008876115204481722, 0.009156782127976888, 0.009464664226080461, 0.00946648189284217, 0.009448261554001126, 0.009762997214986644, 0.00980916055177907, 0.01058147004221259, 0.010750310405128857, 0.010754693666680252, 0.01170912748843261, 0.012786182861672088, 0.013095225975145064, 0.01310799436928179, 0.013918157190860262 ] }, "random_mens": { "ipw": [ 0.004380475594493116, 0.004865456821026282, 0.00535043804755945, 0.0058354192740926155, 0.0063204005006257825, 0.006805381727158949, 0.007290362953692115, 0.007775344180225281, 0.008260325406758449, 0.008745306633291615, 0.009230287859824781, 0.009715269086357947, 0.010200250312891113, 0.010685231539424281, 0.011170212765957446, 0.011655193992490613, 0.012140175219023781, 0.012625156445556946, 0.013110137672090114, 0.01359511889862328, 0.014080100125156446 ] }, "random_womens": { "ipw": [ 0.004380475594493116, 0.004730342089082675, 0.005080208583672234, 0.005430075078261792, 0.005779941572851352, 0.006129808067440909, 0.0064796745620304685, 0.006829541056620026, 0.007179407551209585, 0.007529274045799145, 0.007879140540388702, 0.008229007034978263, 0.008578873529567821, 0.00892874002415738, 0.009278606518746936, 0.009628473013336497, 0.009978339507926055, 0.010328206002515614, 0.010678072497105173, 0.011027938991694733, 0.01137780548628429 ] } }, "spend": { "dr_learner": { "ipw": [ 0.5045791614518148, 0.4944669341976722, 0.6110251319448562, 0.6998170308740039, 0.7768993212369577, 0.9166271059678712, 1.0369493838151804, 1.1379819244910252, 1.1379819244910252, 1.1078911860679965, 1.1362889377775833, 1.0995128300962236, 1.1946927424867118, 1.199212454626887, 1.2562713483812369, 1.3220864297329267, 1.3934647150896224, 1.4553389328618378, 1.6012729128368066, 1.644120221973227, 1.6204891205965062 ], "dr": [ 0.5145506089606136, 0.5056771910215005, 0.5987970003706932, 0.685087092291564, 0.7563669691825855, 0.896418039663109, 1.0176225483768795, 1.1149584007519233, 1.1115824571680162, 1.0828597827405848, 1.1087163744811626, 1.0758324472044671, 1.1750332488410264, 1.1777400603876, 1.2348102511055588, 1.2975161863148192, 1.3723322343917137, 1.4369301430909076, 1.5858703142616903, 1.6235993643400017, 1.5955978734340268 ] }, "causal_forest": { "ipw": [ 0.5045791614518148, 0.5903003169485548, 0.609693780255244, 0.6285439054116897, 0.6640930293165709, 0.7098154664184345, 0.9001784201105496, 0.9385088046467062, 0.9385088046467062, 0.9730410324314369, 1.0159493553350665, 1.1470369679680648, 1.1638576528796905, 1.3000824752886244, 1.417845612033745, 1.462401306652018, 1.541605728326243, 1.559966178889447, 1.546388269002088, 1.670233700791825, 1.6573644892774322 ], "dr": [ 0.5145506089606136, 0.5917322430671997, 0.6105227767651356, 0.6158596100637417, 0.6534389128995193, 0.6875618552144558, 0.8781784772495329, 0.9130841121338851, 0.9125119014029704, 0.9444119803528976, 0.9859797177490636, 1.1234030115236322, 1.1373106810661122, 1.2691781925322625, 1.3827967491490725, 1.4286320329162385, 1.5100941410648578, 1.5322819155604663, 1.5237375844529046, 1.6499349717768412, 1.6330892458101125 ] }, "x_learner": { "ipw": [ 0.5045791614518148, 0.49838698778710294, 0.6725755074141929, 0.7010313897671341, 0.7504572345731416, 0.897505932056592, 1.0928815398924465, 1.0194894588466257, 1.1113292585963128, 1.1893955915124579, 1.1375235721709493, 1.1519853994550544, 1.1574962936838131, 1.1880873165178416, 1.2130090937393687, 1.3322909832739802, 1.4168829939388075, 1.5076448838011351, 1.5842218550151528, 1.6184005158411856, 1.6592565792964395 ], "dr": [ 0.5145506089606136, 0.501425736572167, 0.6573713918970553, 0.6803491053666483, 0.7320273585160828, 0.8776009496691591, 1.0693020547903227, 0.9924658982399499, 1.0852485275806567, 1.1626901394326339, 1.1059162993501999, 1.1199443586384106, 1.119803016320898, 1.1528497351638425, 1.180876510930134, 1.3009254473457494, 1.385707815180939, 1.4829203540513698, 1.5611312369771693, 1.595768198452907, 1.633275415523072 ] }, "t_learner": { "ipw": [ 0.5045791614518148, 0.571461615048736, 0.6794450318196997, 0.6134281356995496, 0.7368564079475904, 0.8739574717773777, 0.8580989992166017, 0.9496542377629142, 1.05158863440585, 1.1268273133655222, 1.2191110771257088, 1.190540989516197, 1.2101643029472624, 1.2133752231592483, 1.1895650911051534, 1.2147700347847528, 1.2334793589399466, 1.3708938483734343, 1.5573002939303804, 1.6585878408640473, 1.6072549247188659 ], "dr": [ 0.5145506089606136, 0.5550400630830048, 0.6578603654477293, 0.5867274938359945, 0.7077551985998021, 0.8461679598525899, 0.82388598705888, 0.9173336052874028, 1.0095412191169282, 1.092532887244185, 1.187937653813919, 1.1632108821129137, 1.179276094146642, 1.1866741558960194, 1.1656182761356417, 1.1861120247941164, 1.2059887586823916, 1.3439684866392865, 1.536430806549374, 1.6339757893902807, 1.5836378303753804 ] }, "logistic": { "ipw": [ 0.5045791614518148, 0.7239471214017521, 0.7198047922278158, 0.8332684871051407, 0.9088160465544523, 0.9466508400463172, 0.9789458963667176, 1.0957826432822821, 1.1788114292647602, 1.31860805004073, 1.3680179089198152, 1.3722388100462235, 1.4813362128158951, 1.591173168065443, 1.7651053744705818, 1.7951914897518402, 1.806876408400151, 1.8551536014469456, 1.925788524620863, 2.0085466597898245, 1.9675688750589109 ], "dr": [ 0.5145506089606136, 0.7064520785140764, 0.6865766359534651, 0.8114834065705765, 0.8751747823460909, 0.9056349418413839, 0.9399162885437806, 1.0656132587208234, 1.1399726269012709, 1.2844641874646157, 1.337257407615856, 1.338749585918183, 1.4508291532681257, 1.558677873775788, 1.7298760431136813, 1.7671699647044583, 1.7815088681196296, 1.8337971485514473, 1.904028644803885, 1.9878419747770044, 1.9471522262088372 ] }, "tarnet": { "ipw": [ 0.5045791614518148, 0.6002690863579473, 0.623418335419274, 0.8972684605757195, 0.8997841051314142, 0.9848764080100125, 1.016284418022528, 1.0834105131414267, 1.1571667709637043, 1.3021933667083854, 1.300064142678348, 1.3775594493116394, 1.4166395494367958, 1.5276783479349185, 1.6991145181476843, 1.751738110137672, 1.764993742177722, 1.7766786608260323, 1.779245932415519, 1.8768178973717142, 1.8431382978723403 ], "dr": [ 0.5145506089606136, 0.5934268716403603, 0.6059504775825713, 0.864250542791281, 0.8688765199797033, 0.9468369258999959, 0.9778391460059556, 1.046889048885298, 1.1244229337402272, 1.2708777630175465, 1.2747813195308193, 1.3505670567848067, 1.390130511822201, 1.5000458940721357, 1.6715209076847153, 1.721655812778325, 1.737104417999066, 1.748659356833413, 1.7497456419683763, 1.8430559392644175, 1.8176407564376669 ] }, "recent_spenders": { "ipw": [ 0.5045791614518148, 0.46513610763454316, 0.6445400500625782, 0.7298826658322902, 0.8319680851063829, 0.8536091989987483, 0.9907493742177722, 1.0895948060075094, 1.19081351689612, 1.2378848560700875, 1.2373357321652063, 1.4328785982478096, 1.3898826658322903, 1.482961514392991, 1.5226032540675845, 1.571897684605757, 1.58386576971214, 1.7260685231539423, 1.8057086983729664, 1.8193820400500629, 1.8431382978723403 ], "dr": [ 0.5145506089606136, 0.473398763021832, 0.6586103576666636, 0.7398111339540325, 0.8219423686434904, 0.8490785070283596, 0.9799000920215917, 1.081787091298286, 1.1748365056871246, 1.2171838868467424, 1.218961354274807, 1.4116644406961485, 1.3799192523100245, 1.4660283201162947, 1.5041993416037882, 1.5528579413903325, 1.5659090776703157, 1.7107891398876647, 1.7849123919333783, 1.7948549024701599, 1.8176407564376669 ] }, "likely_buyers": { "ipw": [ 0.5045791614518148, 0.7478066332916145, 0.8338642052565707, 0.8636608260325407, 0.9192756570713391, 0.9395118898623279, 0.9537781602002501, 0.9900281602002502, 1.0927049436795995, 1.1312969336670835, 1.114580725907384, 1.2071198372966205, 1.2550187734668334, 1.3828911138923656, 1.411203066332916, 1.406497183979975, 1.557828535669587, 1.6339173967459322, 1.640301939924906, 1.6590112640800998, 1.8431382978723403 ], "dr": [ 0.5145506089606136, 0.7318385237107117, 0.8164973229688659, 0.8563816177538558, 0.8989320386734524, 0.914812464383621, 0.931181560935878, 0.9608292216067851, 1.0612006053208431, 1.0960525235868595, 1.079389726691987, 1.1757661929991494, 1.22215970031248, 1.3480261934088011, 1.3769234196441813, 1.3733985120961427, 1.5272455937008784, 1.6020657758758003, 1.6085266982201423, 1.6308333488923075, 1.8176407564376669 ] }, "random_mens": { "ipw": [ 0.5045791614518148, 0.5715071182728411, 0.6384350750938674, 0.7053630319148936, 0.77229098873592, 0.8392189455569463, 0.9061469023779725, 0.9730748591989988, 1.0400028160200252, 1.1069307728410513, 1.1738587296620777, 1.240786686483104, 1.3077146433041302, 1.3746426001251566, 1.4415705569461827, 1.5084985137672091, 1.5754264705882355, 1.6423544274092616, 1.709282384230288, 1.7762103410513141, 1.8431382978723405 ] }, "random_womens": { "ipw": [ 0.5045791614518148, 0.537375218965259, 0.5701712764787031, 0.6029673339921473, 0.6357633915055916, 0.6685594490190356, 0.7013555065324798, 0.734151564045924, 0.7669476215593682, 0.7997436790728124, 0.8325397365862566, 0.8653357940997007, 0.8981318516131449, 0.9309279091265891, 0.9637239666400332, 0.9965200241534774, 1.0293160816669216, 1.0621121391803656, 1.09490819669381, 1.1277042542072542, 1.1605003117206982 ] } } }, "compare": { "conversion": [ { "k": 0.1, "random_mens": 0.0053504380475594485, "dr_learner": { "value": 0.00531739331271321, "vs_random": -3.304473484623855e-05, "vs_random_ci": [ -0.001081006633467041, 0.0010933779034530743 ], "prob_better": 0.4755, "dr_value": 0.005149857190940809 }, "recent_spenders": { "value": 0.00578848560700876, "vs_random": 0.0004380475594493118, "vs_random_ci": [ -0.0009390643856022506, 0.0018617020716192201 ], "prob_better": 0.7055, "dr_value": 0.005889497591470643 }, "likely_buyers": { "value": 0.006727158948685856, "vs_random": 0.0013767209011264082, "vs_random_ci": [ 3.089792298851537e-05, 0.0028629538137465715 ], "prob_better": 0.979, "dr_value": 0.006520300682861268 }, "causal_forest": { "value": 0.0056302844266055765, "vs_random": 0.00027984637904612705, "vs_random_ci": [ -0.0009398437105119228, 0.0015004687797045317 ], "prob_better": 0.656, "dr_value": 0.005602797880820658 }, "x_learner": { "value": 0.005473838869659393, "vs_random": 0.00012340082209994426, "vs_random_ci": [ -0.0009549030233756639, 0.0012825862853787823 ], "prob_better": 0.5845, "dr_value": 0.005295190707078858 }, "t_learner": { "value": 0.0056302844266055765, "vs_random": 0.0002798463790461269, "vs_random_ci": [ -0.0009560880367644131, 0.0015636029769666476 ], "prob_better": 0.6725, "dr_value": 0.005359135297708859 }, "logistic": { "value": 0.005786144775732758, "vs_random": 0.0004357067281733092, "vs_random_ci": [ -0.0009882949118036777, 0.0018753822630969788 ], "prob_better": 0.7165, "dr_value": 0.00552416270860561 }, "tarnet": { "value": 0.005944931163954944, "vs_random": 0.0005944931163954943, "vs_random_ci": [ -0.0006883604219183326, 0.002018147613853216 ], "prob_better": 0.8115, "dr_value": 0.005757472722015442 } }, { "k": 0.2, "random_mens": 0.006320400500625782, "dr_learner": { "value": 0.006411927003517489, "vs_random": 9.152650289170718e-05, "vs_random_ci": [ -0.0012828681705286727, 0.0015321345534175595 ], "prob_better": 0.5465, "dr_value": 0.006205865726678005 }, "recent_spenders": { "value": 0.007509386733416771, "vs_random": 0.0011889862327909887, "vs_random_ci": [ -0.0004693366354331374, 0.0027221525087952614 ], "prob_better": 0.919, "dr_value": 0.007449673798985704 }, "likely_buyers": { "value": 0.008291614518147684, "vs_random": 0.0019712140175219025, "vs_random_ci": [ 0.0003128910313535016, 0.003692115191370249 ], "prob_better": 0.9875, "dr_value": 0.00811825928932483 }, "causal_forest": { "value": 0.00703829443912122, "vs_random": 0.0007178939384954387, "vs_random_ci": [ -0.0008454057606286369, 0.0023750608961563557 ], "prob_better": 0.807, "dr_value": 0.0068828731702371 }, "x_learner": { "value": 0.006881848882175038, "vs_random": 0.000561448381549256, "vs_random_ci": [ -0.0008459909207886085, 0.0020308662205934525 ], "prob_better": 0.777, "dr_value": 0.006672708410432364 }, "t_learner": { "value": 0.00766232104344895, "vs_random": 0.0013419205428231674, "vs_random_ci": [ -0.0002515124935598578, 0.003027465345803648 ], "prob_better": 0.9525, "dr_value": 0.007360638628691767 }, "logistic": { "value": 0.007818181392576132, "vs_random": 0.0014977808919503495, "vs_random_ci": [ -0.00022385860211215908, 0.003343991731526326 ], "prob_better": 0.95, "dr_value": 0.007562099811972513 }, "tarnet": { "value": 0.0076658322903629525, "vs_random": 0.0013454317897371717, "vs_random_ci": [ -0.00034418021241435783, 0.003066332550952211 ], "prob_better": 0.939, "dr_value": 0.00741959351691313 } }, { "k": 0.3, "random_mens": 0.007290362953692114, "dr_learner": { "value": 0.008132828129925499, "vs_random": 0.0008424651762333838, "vs_random_ci": [ -0.0007677681525819935, 0.002500205644173541 ], "prob_better": 0.8455, "dr_value": 0.007967904637223416 }, "recent_spenders": { "value": 0.007978723404255319, "vs_random": 0.0006883604505632041, "vs_random_ci": [ -0.0010172870999667794, 0.0024409417121205455 ], "prob_better": 0.7615, "dr_value": 0.007901769041964012 }, "likely_buyers": { "value": 0.009073842302878598, "vs_random": 0.001783479349186483, "vs_random_ci": [ -3.128888783976436e-05, 0.003566958714509383 ], "prob_better": 0.974, "dr_value": 0.008876115204481722 }, "causal_forest": { "value": 0.008445719243817864, "vs_random": 0.0011553562901257498, "vs_random_ci": [ -0.0004731657987576909, 0.0029064199188724156 ], "prob_better": 0.91, "dr_value": 0.008224877858240448 }, "x_learner": { "value": 0.009069745848145594, "vs_random": 0.0017793828944534784, "vs_random_ci": [ 9.056477865669878e-05, 0.0034991283493582158 ], "prob_better": 0.981, "dr_value": 0.008843658980379988 }, "t_learner": { "value": 0.009069745848145594, "vs_random": 0.0017793828944534784, "vs_random_ci": [ -2.3554763174615793e-06, 0.0034960227785631993 ], "prob_better": 0.974, "dr_value": 0.00872366364486256 }, "logistic": { "value": 0.007974626949522315, "vs_random": 0.0006842639958301992, "vs_random_ci": [ -0.0011151925369631498, 0.002499756234465163 ], "prob_better": 0.764, "dr_value": 0.007700192159301452 }, "tarnet": { "value": 0.008291614518147684, "vs_random": 0.0010012515644555694, "vs_random_ci": [ -0.0007509387214668095, 0.0026286765467375497 ], "prob_better": 0.8655, "dr_value": 0.007979915896522775 } }, { "k": 0.4, "random_mens": 0.008260325406758447, "dr_learner": { "value": 0.008915055914656413, "vs_random": 0.0006547305078979653, "vs_random_ci": [ -0.0009705768708954566, 0.0023143771395552864 ], "prob_better": 0.775, "dr_value": 0.008672173366053979 }, "recent_spenders": { "value": 0.009230287859824781, "vs_random": 0.0009699624530663335, "vs_random_ci": [ -0.0007822276675142348, 0.002659574622521177 ], "prob_better": 0.853, "dr_value": 0.009128136323653488 }, "likely_buyers": { "value": 0.00969962453066333, "vs_random": 0.0014392991239048818, "vs_random_ci": [ -0.00028160202855360695, 0.00312891136854887 ], "prob_better": 0.944, "dr_value": 0.009464664226080461 }, "causal_forest": { "value": 0.008758025149891229, "vs_random": 0.0004976997431327821, "vs_random_ci": [ -0.0011903829436050728, 0.0023101416532881557 ], "prob_better": 0.7085, "dr_value": 0.008554053727797339 }, "x_learner": { "value": 0.009538497311165139, "vs_random": 0.0012781719044066937, "vs_random_ci": [ -0.00047110684026847595, 0.0030623168975580446 ], "prob_better": 0.919, "dr_value": 0.009291411966336656 }, "t_learner": { "value": 0.010007248774184688, "vs_random": 0.0017469233674262412, "vs_random_ci": [ -9.73931350017665e-05, 0.003685722005320713 ], "prob_better": 0.9665, "dr_value": 0.009656544038784057 }, "logistic": { "value": 0.009069160640326594, "vs_random": 0.0008088352335681457, "vs_random_ci": [ -0.0010692644747905434, 0.002623215696075927 ], "prob_better": 0.8005, "dr_value": 0.008803630004261497 }, "tarnet": { "value": 0.00969962453066333, "vs_random": 0.0014392991239048818, "vs_random_ci": [ -0.000251095160638215, 0.0031289111357182264 ], "prob_better": 0.949, "dr_value": 0.009463375650724005 } }, { "k": 0.5, "random_mens": 0.009230287859824781, "dr_learner": { "value": 0.009226776612910777, "vs_random": -3.51124691400385e-06, "vs_random_ci": [ -0.0017232417623745278, 0.0017203988769324396 ], "prob_better": 0.4985, "dr_value": 0.008995909363574124 }, "recent_spenders": { "value": 0.009386733416770963, "vs_random": 0.00015644555694618276, "vs_random_ci": [ -0.0014099656982580198, 0.001799123827368021 ], "prob_better": 0.547, "dr_value": 0.009240375429333803 }, "likely_buyers": { "value": 0.00969962453066333, "vs_random": 0.0004693366708385482, "vs_random_ci": [ -0.0012515643611550331, 0.0021120149176567793 ], "prob_better": 0.7, "dr_value": 0.009448261554001126 }, "causal_forest": { "value": 0.009540252934622143, "vs_random": 0.00030996507479736236, "vs_random_ci": [ -0.0014109506388194858, 0.0021085184591356663 ], "prob_better": 0.6135, "dr_value": 0.009285756950778159 }, "x_learner": { "value": 0.010006663566365688, "vs_random": 0.0007763757065409071, "vs_random_ci": [ -0.0010986448760377243, 0.002652566839242354 ], "prob_better": 0.7805, "dr_value": 0.009716166033826036 }, "t_learner": { "value": 0.0107877209354586, "vs_random": 0.0015574330756338193, "vs_random_ci": [ -0.00024414361687377087, 0.0035930635407566925 ], "prob_better": 0.949, "dr_value": 0.010470238650520809 }, "logistic": { "value": 0.010789476558915602, "vs_random": 0.0015591886990908212, "vs_random_ci": [ -0.0003934691980248317, 0.0033600684255361557 ], "prob_better": 0.944, "dr_value": 0.010547880511209378 }, "tarnet": { "value": 0.01095118898623279, "vs_random": 0.0017209011264080099, "vs_random_ci": [ 7.822283077985048e-05, 0.003363579511642456 ], "prob_better": 0.9785, "dr_value": 0.01078148055942402 } }, { "k": 0.6, "random_mens": 0.010200250312891113, "dr_learner": { "value": 0.010321310303715055, "vs_random": 0.0001210599908239424, "vs_random_ci": [ -0.00172442706534639, 0.0019335022254381327 ], "prob_better": 0.5475, "dr_value": 0.01017873348459452 }, "recent_spenders": { "value": 0.010638297872340425, "vs_random": 0.00043804755944931256, "vs_random_ci": [ -0.001032540577580221, 0.0019712136185262353 ], "prob_better": 0.697, "dr_value": 0.010532841166369874 }, "likely_buyers": { "value": 0.010012515644555693, "vs_random": -0.00018773466833541834, "vs_random_ci": [ -0.001689612166956067, 0.0013149249396519694 ], "prob_better": 0.3865, "dr_value": 0.00980916055177907 }, "causal_forest": { "value": 0.010790061766734601, "vs_random": 0.0005898114538434899, "vs_random_ci": [ -0.0012243716075317935, 0.002467801643069833 ], "prob_better": 0.721, "dr_value": 0.01058613103969128 }, "x_learner": { "value": 0.010474829821566235, "vs_random": 0.0002745795086751218, "vs_random_ci": [ -0.0016021819319576025, 0.002181489171925932 ], "prob_better": 0.6095, "dr_value": 0.010163664575161274 }, "t_learner": { "value": 0.0107871357276396, "vs_random": 0.0005868854147484866, "vs_random_ci": [ -0.001350127931800671, 0.002619631006382402 ], "prob_better": 0.727, "dr_value": 0.010465927837191566 }, "logistic": { "value": 0.01125822802193515, "vs_random": 0.0010579777090440367, "vs_random_ci": [ -0.000786353464354761, 0.0029342121793888504 ], "prob_better": 0.8675, "dr_value": 0.011033144099305276 }, "tarnet": { "value": 0.011733416770963704, "vs_random": 0.0015331664580725917, "vs_random_ci": [ 3.0506836446875287e-05, 0.0029419587459415153 ], "prob_better": 0.9755, "dr_value": 0.011562279351377868 } } ], "spend": [ { "k": 0.1, "random_mens": 0.6384350750938673, "dr_learner": { "value": 0.6110251319448562, "vs_random": -0.027409943149011077, "vs_random_ci": [ -0.16069984026253223, 0.13551456183195107 ], "prob_better": 0.351, "dr_value": 0.5987970003706932 }, "recent_spenders": { "value": 0.6445400500625782, "vs_random": 0.006104974968710876, "vs_random_ci": [ -0.26840866804122926, 0.25750895664095874 ], "prob_better": 0.506, "dr_value": 0.6586103576666636 }, "likely_buyers": { "value": 0.8338642052565707, "vs_random": 0.19542913016270336, "vs_random_ci": [ -0.0026991676539182647, 0.4313068211078641 ], "prob_better": 0.971, "dr_value": 0.8164973229688659 }, "causal_forest": { "value": 0.609693780255244, "vs_random": -0.028741294838623095, "vs_random_ci": [ -0.15835220627486707, 0.12233764939010136 ], "prob_better": 0.3275, "dr_value": 0.6105227767651356 }, "x_learner": { "value": 0.6725755074141929, "vs_random": 0.034140432320325584, "vs_random_ci": [ -0.13182576820254327, 0.23848160505294785 ], "prob_better": 0.6315, "dr_value": 0.6573713918970553 }, "t_learner": { "value": 0.6794450318196997, "vs_random": 0.041009956725832464, "vs_random_ci": [ -0.12684680633246898, 0.2490604583173989 ], "prob_better": 0.659, "dr_value": 0.6578603654477293 }, "logistic": { "value": 0.7198047922278158, "vs_random": 0.08136971713394858, "vs_random_ci": [ -0.1971242416650057, 0.36572840213775565 ], "prob_better": 0.7025, "dr_value": 0.6865766359534651 }, "tarnet": { "value": 0.623418335419274, "vs_random": -0.015016739674593277, "vs_random_ci": [ -0.25207012221217157, 0.21337300166487647 ], "prob_better": 0.44, "dr_value": 0.6059504775825713 } }, { "k": 0.2, "random_mens": 0.77229098873592, "dr_learner": { "value": 0.7768993212369577, "vs_random": 0.004608332501037703, "vs_random_ci": [ -0.1901474256068468, 0.24710573889315113 ], "prob_better": 0.5245, "dr_value": 0.7563669691825855 }, "recent_spenders": { "value": 0.8319680851063829, "vs_random": 0.059677096370463, "vs_random_ci": [ -0.24716665409505367, 0.3424407795071602 ], "prob_better": 0.6415, "dr_value": 0.8219423686434904 }, "likely_buyers": { "value": 0.9192756570713391, "vs_random": 0.14698466833541918, "vs_random_ci": [ -0.12706276439130307, 0.43756380155682556 ], "prob_better": 0.8535, "dr_value": 0.8989320386734524 }, "causal_forest": { "value": 0.6640930293165709, "vs_random": -0.10819795941934904, "vs_random_ci": [ -0.3330304339528084, 0.1026420377194881 ], "prob_better": 0.1825, "dr_value": 0.6534389128995193 }, "x_learner": { "value": 0.7504572345731416, "vs_random": -0.02183375416277834, "vs_random_ci": [ -0.20765269547700882, 0.19644290320575233 ], "prob_better": 0.413, "dr_value": 0.7320273585160828 }, "t_learner": { "value": 0.7368564079475904, "vs_random": -0.03543458078832961, "vs_random_ci": [ -0.2740346610546112, 0.22029190473258484 ], "prob_better": 0.4115, "dr_value": 0.7077551985998021 }, "logistic": { "value": 0.9088160465544523, "vs_random": 0.13652505781853244, "vs_random_ci": [ -0.16214036308228968, 0.4596564888954162 ], "prob_better": 0.798, "dr_value": 0.8751747823460909 }, "tarnet": { "value": 0.8997841051314142, "vs_random": 0.12749311639549427, "vs_random_ci": [ -0.167131981998682, 0.44358384385704985 ], "prob_better": 0.795, "dr_value": 0.8688765199797033 } }, { "k": 0.3, "random_mens": 0.9061469023779724, "dr_learner": { "value": 1.0369493838151804, "vs_random": 0.130802481437208, "vs_random_ci": [ -0.13228221014142036, 0.4120815068483352 ], "prob_better": 0.8395, "dr_value": 1.0176225483768795 }, "recent_spenders": { "value": 0.9907493742177722, "vs_random": 0.0846024718397998, "vs_random_ci": [ -0.24787316657602787, 0.4108123898506164 ], "prob_better": 0.6825, "dr_value": 0.9799000920215917 }, "likely_buyers": { "value": 0.9537781602002501, "vs_random": 0.04763125782227791, "vs_random_ci": [ -0.2767928786575794, 0.347732034325599 ], "prob_better": 0.6115, "dr_value": 0.931181560935878 }, "causal_forest": { "value": 0.9001784201105496, "vs_random": -0.0059684822674227435, "vs_random_ci": [ -0.268347080796957, 0.2533206008374691 ], "prob_better": 0.4885, "dr_value": 0.8781784772495329 }, "x_learner": { "value": 1.0928815398924465, "vs_random": 0.1867346375144742, "vs_random_ci": [ -0.06259942762553691, 0.45607948526740066 ], "prob_better": 0.916, "dr_value": 1.0693020547903227 }, "t_learner": { "value": 0.8580989992166017, "vs_random": -0.04804790316137059, "vs_random_ci": [ -0.352768125385046, 0.22785985954105845 ], "prob_better": 0.3685, "dr_value": 0.82388598705888 }, "logistic": { "value": 0.9789458963667176, "vs_random": 0.07279899398874534, "vs_random_ci": [ -0.2189721893519163, 0.37574789673089964 ], "prob_better": 0.659, "dr_value": 0.9399162885437806 }, "tarnet": { "value": 1.016284418022528, "vs_random": 0.11013751564455572, "vs_random_ci": [ -0.19911779910326005, 0.4209585666656494 ], "prob_better": 0.7485, "dr_value": 0.9778391460059556 } }, { "k": 0.4, "random_mens": 1.040002816020025, "dr_learner": { "value": 1.1379819244910252, "vs_random": 0.09797910847100019, "vs_random_ci": [ -0.1723245158791542, 0.3928734138607979 ], "prob_better": 0.772, "dr_value": 1.1115824571680162 }, "recent_spenders": { "value": 1.19081351689612, "vs_random": 0.15081070087609508, "vs_random_ci": [ -0.1664432018995285, 0.44919121712446186 ], "prob_better": 0.814, "dr_value": 1.1748365056871246 }, "likely_buyers": { "value": 1.0927049436795995, "vs_random": 0.0527021276595744, "vs_random_ci": [ -0.26208784282207487, 0.35561214238405225 ], "prob_better": 0.6235, "dr_value": 1.0612006053208431 }, "causal_forest": { "value": 0.9385088046467062, "vs_random": -0.10149401137331891, "vs_random_ci": [ -0.3812349073588848, 0.17862807884812332 ], "prob_better": 0.249, "dr_value": 0.9125119014029704 }, "x_learner": { "value": 1.1113292585963128, "vs_random": 0.07132644257628766, "vs_random_ci": [ -0.2159693729132414, 0.3753024436533451 ], "prob_better": 0.6995, "dr_value": 1.0852485275806567 }, "t_learner": { "value": 1.05158863440585, "vs_random": 0.01158581838582517, "vs_random_ci": [ -0.30248436108231547, 0.2999233864247799 ], "prob_better": 0.536, "dr_value": 1.0095412191169282 }, "logistic": { "value": 1.1788114292647602, "vs_random": 0.13880861324473542, "vs_random_ci": [ -0.15771872103214263, 0.4511932037770747 ], "prob_better": 0.8105, "dr_value": 1.1399726269012709 }, "tarnet": { "value": 1.1571667709637043, "vs_random": 0.1171639549436795, "vs_random_ci": [ -0.1727142371237278, 0.42765466123819307 ], "prob_better": 0.7625, "dr_value": 1.1244229337402272 } }, { "k": 0.5, "random_mens": 1.1738587296620775, "dr_learner": { "value": 1.1362889377775833, "vs_random": -0.03756979188449402, "vs_random_ci": [ -0.3279407165944576, 0.27875520586967445 ], "prob_better": 0.4235, "dr_value": 1.1087163744811626 }, "recent_spenders": { "value": 1.2373357321652063, "vs_random": 0.0634770025031289, "vs_random_ci": [ -0.23145004622638227, 0.3363305345177648 ], "prob_better": 0.659, "dr_value": 1.218961354274807 }, "likely_buyers": { "value": 1.114580725907384, "vs_random": -0.05927800375469335, "vs_random_ci": [ -0.35853757336735725, 0.2348326116800307 ], "prob_better": 0.3295, "dr_value": 1.079389726691987 }, "causal_forest": { "value": 1.0159493553350665, "vs_random": -0.15790937432701094, "vs_random_ci": [ -0.4725100569427013, 0.1409288227558132 ], "prob_better": 0.152, "dr_value": 0.9859797177490636 }, "x_learner": { "value": 1.1375235721709493, "vs_random": -0.036335157491128214, "vs_random_ci": [ -0.33777825310826304, 0.24731082431972015 ], "prob_better": 0.419, "dr_value": 1.1059162993501999 }, "t_learner": { "value": 1.2191110771257088, "vs_random": 0.045252347463631305, "vs_random_ci": [ -0.26040062457323077, 0.33062829747796035 ], "prob_better": 0.623, "dr_value": 1.187937653813919 }, "logistic": { "value": 1.3680179089198152, "vs_random": 0.194159179257738, "vs_random_ci": [ -0.0955072071403265, 0.4996936686336994 ], "prob_better": 0.904, "dr_value": 1.337257407615856 }, "tarnet": { "value": 1.300064142678348, "vs_random": 0.1262054130162703, "vs_random_ci": [ -0.1663244754076004, 0.41855172812938674 ], "prob_better": 0.7995, "dr_value": 1.2747813195308193 } }, { "k": 0.6, "random_mens": 1.30771464330413, "dr_learner": { "value": 1.1946927424867118, "vs_random": -0.11302190081741822, "vs_random_ci": [ -0.4260948456823826, 0.18493955284357036 ], "prob_better": 0.2425, "dr_value": 1.1750332488410264 }, "recent_spenders": { "value": 1.3898826658322903, "vs_random": 0.08216802252816018, "vs_random_ci": [ -0.19198045022785662, 0.3488493867218494 ], "prob_better": 0.712, "dr_value": 1.3799192523100245 }, "likely_buyers": { "value": 1.2550187734668334, "vs_random": -0.05269586983729663, "vs_random_ci": [ -0.32419858276844027, 0.20944656655192362 ], "prob_better": 0.3445, "dr_value": 1.22215970031248 }, "causal_forest": { "value": 1.1638576528796905, "vs_random": -0.1438569904244395, "vs_random_ci": [ -0.45444913282990457, 0.15067215487360922 ], "prob_better": 0.161, "dr_value": 1.1373106810661122 }, "x_learner": { "value": 1.1574962936838131, "vs_random": -0.15021834962031708, "vs_random_ci": [ -0.4575677327811718, 0.12976216040551636 ], "prob_better": 0.164, "dr_value": 1.119803016320898 }, "t_learner": { "value": 1.2101643029472624, "vs_random": -0.09755034035686748, "vs_random_ci": [ -0.4058604657649994, 0.19197040721774097 ], "prob_better": 0.267, "dr_value": 1.179276094146642 }, "logistic": { "value": 1.4813362128158951, "vs_random": 0.17362156951176502, "vs_random_ci": [ -0.11711063645780086, 0.4647871650755405 ], "prob_better": 0.873, "dr_value": 1.4508291532681257 }, "tarnet": { "value": 1.4166395494367958, "vs_random": 0.10892490613266581, "vs_random_ci": [ -0.16864092350006102, 0.3819527864456169 ], "prob_better": 0.7805, "dr_value": 1.390130511822201 } } ] }, "blanket": { "conversion": { "nobody": { "value": 0.004380475594493116, "ci": [ 0.002816020278260112, 0.006101376377046108 ] }, "everyone_mens": { "value": 0.014080100125156446, "ci": [ 0.011264079250395298, 0.01705256476998329 ] }, "everyone_womens": { "value": 0.011377805486284288, "ci": [ 0.008884039707481861, 0.014183291234076023 ] } }, "spend": { "nobody": { "value": 0.5045791614518148, "ci": [ 0.26689970046281813, 0.8058126524090763 ] }, "everyone_mens": { "value": 1.8431382978723403, "ci": [ 1.3371176302433014, 2.3817509353160853 ] }, "everyone_womens": { "value": 1.1605003117206982, "ci": [ 0.8250344350934029, 1.5456452488899228 ] } } }, "calibration_fifths": [ { "fifth": 1, "customers": 3840, "predicted_mens": 0.0017000329546354167, "predicted_womens": -0.00041781008260416663, "observed_mens": 0.008086507828389652, "observed_mens_ci": [ 0.0006224222895201111, 0.015550593367259193 ], "observed_womens": 0.00485416067309682, "observed_womens_ci": [ -0.0020895121662008543, 0.011797833512394496 ], "base_rate": 0.005529225908372828 }, { "fifth": 2, "customers": 3840, "predicted_mens": 0.0035437022905208332, "predicted_womens": 0.000227901365390625, "observed_mens": 0.006567803163879558, "observed_mens_ci": [ -0.0009822637929868972, 0.014117870120746013 ], "observed_womens": 0.005363599959357611, "observed_womens_ci": [ -0.0018622138344124807, 0.012589413753127703 ], "base_rate": 0.006201550387596899 }, { "fifth": 3, "customers": 3840, "predicted_mens": 0.0049143877056249995, "predicted_womens": 0.001540763230625, "observed_mens": 0.006557626522181606, "observed_mens_ci": [ 0.000683130175361946, 0.012432122869001265 ], "observed_womens": 0.008192912605787283, "observed_womens_ci": [ 0.0020916625907835542, 0.014294162620791011 ], "base_rate": 0.0023492560689115116 }, { "fifth": 4, "customers": 3840, "predicted_mens": 0.006957844526666667, "predicted_womens": 0.0017456830985677083, "observed_mens": 0.014263897709626619, "observed_mens_ci": [ 0.006754029398709772, 0.021773766020543466 ], "observed_womens": 0.007808721933066201, "observed_womens_ci": [ 0.0017316827136840476, 0.013885761152448354 ], "base_rate": 0.0023237800154918666 }, { "fifth": 5, "customers": 3840, "predicted_mens": 0.010751720567213541, "predicted_womens": 0.004192052696901041, "observed_mens": 0.0127165773350081, "observed_mens_ci": [ 0.004416049781877183, 0.02101710488813902 ], "observed_womens": 0.008810705459221604, "observed_womens_ci": [ 0.001075195305094115, 0.016546215613349093 ], "base_rate": 0.005520504731861199 } ], "persuasion_at_30": { "likely_buyers": { "customers": 5760, "no_email_rate": 0.008483563096500531, "mens_email_rate": 0.02354145342886387, "womens_email_rate": 0.019270833333333334, "mean_recency": 4.4300347222222225, "mean_history": 393.04871701388896, "share_newbie": 0.3546875 }, "dr_learner": { "customers": 5760, "no_email_rate": 0.004714510214772132, "mens_email_rate": 0.019003595274781716, "womens_email_rate": 0.014705882352941176, "mean_recency": 5.870486111111111, "mean_history": 411.6582916666666, "share_newbie": 0.5973958333333333 }, "not_picked_by_either": { "customers": 11021, "no_email_rate": 0.0032537960954446853, "mens_email_rate": 0.009058468295360967, "womens_email_rate": 0.007859078590785908, "mean_recency": 6.1362852735686415, "mean_history": 152.81337991107884, "share_newbie": 0.5376100172398149 }, "overlap_share": 0.5800347222222222 }, "email_mix_at_30": { "mens": 4803, "womens": 957 }, "qini": { "dr_learner": { "mens": 0.04870889965587812, "womens": 0.10098248648870686 }, "causal_forest": { "mens": 0.06165300092531757, "womens": 0.09371050960875062 }, "x_learner": { "mens": 0.05196846754631873, "womens": 0.09374876713926024 }, "t_learner": { "mens": 0.08323841894527118, "womens": 0.12248404554781574 }, "logistic": { "mens": 0.05597671213924141, "womens": 0.1742868720996284 }, "tarnet": { "mens": 0.11957129762552952, "womens": 0.13660776345172881 } }, "test_size_needed_for_1_per_1000": 104617.00639621992, "se_per_1000_at_30": 0.8336667848866159, "single_level": { "comparisons": 48, "cleared_zero": [ { "k": 0.1, "rule": "likely_buyers" }, { "k": 0.2, "rule": "likely_buyers" }, { "k": 0.3, "rule": "x_learner" }, { "k": 0.5, "rule": "tarnet" }, { "k": 0.6, "rule": "tarnet" } ], "expected_by_chance": 1.2000000000000002, "chosen_cleared_zero_anywhere": false }, "dr_check": { "comparisons": 48, "sign_flips": 4, "largest_flip_size": 0.2745795086751216, "correlation": 0.9897338439715903 } }, "assumptions": { "defaults": { "margin": 0.4, "cost_per_email": 0.01, "list_size": 100000 }, "note": "Margin, cost per email and list size are not in the data. Change them in the dashboard.", "money": { "mens": { "extra_spend_per_email": 0.7698271558945368, "net_per_1000_emails": 297.93086235781476, "break_even_cost_per_email": 0.30793086235781475 }, "womens": { "extra_spend_per_email": 0.4244122159365967, "net_per_1000_emails": 159.7648863746387, "break_even_cost_per_email": 0.1697648863746387 } } }, "headline": { "mens_rate": 0.01253109306800582, "womens_rate": 0.008837144059475383, "none_rate": 0.005726086548390125, "mens_extra_buyers_per_1000": 6.805006519615695, "womens_extra_buyers_per_1000": 3.111057511085258, "mens_extra_spend_per_customer": 0.7698271558945368, "womens_extra_spend_per_customer": 0.4244122159365967, "mens_anyway_share": 0.45695028496834605, "womens_anyway_share": 0.6479566825948126, "everyone_mens_extra_buyers_per_100k": 680.5006519615695, "everyone_mens_extra_spend_per_100k": 76982.71558945367, "models_tested": 6 } }