Small refactor to ease up understanding on what parameters need to be changed when monitoring A/B test
This commit is contained in:
uri
parent
a5e30354e4
commit
814a5bce28
1 changed files with 52 additions and 39 deletions
|
|
@ -12,7 +12,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 47,
|
||||
"execution_count": 23,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -29,7 +29,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 48,
|
||||
"execution_count": 24,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -47,7 +47,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 49,
|
||||
"execution_count": 25,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -79,7 +79,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 50,
|
||||
"execution_count": 26,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -97,6 +97,35 @@
|
|||
"print(test_df.head())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## A/B test configuration\n",
|
||||
"In this section we configure the parameters for the A/B test. Likely you do NOT need to change anything else than this, unless of course you want to create new metrics and so on.\n",
|
||||
"\n",
|
||||
"The parameters to be specified are:\n",
|
||||
"* **ab_test_name**: this should be the name of the feature flag corresponding to the A/B test. If you don't know the name, ask Guest Squad\n",
|
||||
"* **var_A** and **var_B**: these correspond to the name of the variants. At this moment, we can only handle univariant testing (though updating the code to include multivariant testing should not be extremely difficult). In general, choose var_A to be the Control group."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 27,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# A/B test name to measure\n",
|
||||
"#ab_test_name = \"AAVariantTest\"\n",
|
||||
"ab_test_name = \"VerificationProductSelectionButtonPosition\"\n",
|
||||
"\n",
|
||||
"# Define the variations in which we want to run the tests\n",
|
||||
"var_A = 'Fixed' # Ideally, this should be the control group\n",
|
||||
"var_B = 'Relative' # Ideally, this should be the study group\n",
|
||||
"\n",
|
||||
"variations = [var_A, var_B]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
|
|
@ -107,7 +136,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 51,
|
||||
"execution_count": 28,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -159,10 +188,6 @@
|
|||
}
|
||||
],
|
||||
"source": [
|
||||
"# A/B test name to measure\n",
|
||||
"#ab_test_name = \"AAVariantTest\"\n",
|
||||
"ab_test_name = \"VerificationProductSelectionButtonPosition\"\n",
|
||||
"\n",
|
||||
"# Query to extract data\n",
|
||||
"data_extraction_query = \"\"\"\n",
|
||||
"select \n",
|
||||
|
|
@ -225,7 +250,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 52,
|
||||
"execution_count": 29,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -298,7 +323,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 53,
|
||||
"execution_count": 30,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -376,7 +401,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 54,
|
||||
"execution_count": 31,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -466,15 +491,10 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 55,
|
||||
"execution_count": 32,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Define the variations in which we want to run the tests\n",
|
||||
"var_A = 'Fixed'\n",
|
||||
"var_B = 'Relative'\n",
|
||||
"variations = [var_A, var_B]\n",
|
||||
"\n",
|
||||
"# Define the Z-test metric definitions (with both success_counts and total_counts)\n",
|
||||
"z_stat_metric_definition = {\n",
|
||||
" 'conversion_rate': {\n",
|
||||
|
|
@ -542,7 +562,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 56,
|
||||
"execution_count": 33,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -590,7 +610,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 57,
|
||||
"execution_count": 34,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -606,19 +626,19 @@
|
|||
"\n",
|
||||
"Main Metrics - Comparing Relative vs. Fixed.\n",
|
||||
"\n",
|
||||
"CONVERSION RATE (not significant): 41.7% vs. 16.7% (25.0% | 150.0%).\n",
|
||||
"PAYMENT RATE (not significant): 20.8% vs. 22.2% (-1.4% | -6.2%).\n",
|
||||
"AVG GUEST REVENUE PER GJ (not significant): 5.9 vs. 5.8 (0.1 | 1.8%).\n",
|
||||
"CONVERSION RATE (not significant): 41.7% vs. 16.7% (25.0% ppts.| 150.0%).\n",
|
||||
"PAYMENT RATE (not significant): 20.8% vs. 22.2% (-1.4% ppts.| -6.2%).\n",
|
||||
"AVG GUEST REVENUE PER GJ (not significant): 5.9 vs. 5.8 (0.1 ppts.| 1.8%).\n",
|
||||
"\n",
|
||||
"Other Metrics\n",
|
||||
"\n",
|
||||
"WAIVER PAYMENT RATE (not significant): 16.7% vs. 16.7% (0.0% | 0.0%).\n",
|
||||
"DEPOSIT PAYMENT RATE (not significant): 4.2% vs. 5.6% (-1.4% | -25.0%).\n",
|
||||
"CIH PAYMENT RATE (not significant): 0.0% vs. 5.6% (-5.6% | -100.0%).\n",
|
||||
"AVG WAIVER REVENUE PER GJ (not significant): 5.64 vs. 4.91 (0.73 | 14.9%).\n",
|
||||
"AVG DEPOSIT REVENUE PER GJ (not significant): 0.26 vs. 0.37 (-0.11 | -28.9%).\n",
|
||||
"AVG CIH REVENUE PER GJ (not significant): 0.0 vs. 0.52 (-0.52 | -100.0%).\n",
|
||||
"AVG CSAT PER GJ WITH RESPONSE (not significant): 3.0 vs. 4.0 (-1.0 | -25.0%).\n"
|
||||
"WAIVER PAYMENT RATE (not significant): 16.7% vs. 16.7% (0.0% ppts.| 0.0%).\n",
|
||||
"DEPOSIT PAYMENT RATE (not significant): 4.2% vs. 5.6% (-1.4% ppts.| -25.0%).\n",
|
||||
"CIH PAYMENT RATE (not significant): 0.0% vs. 5.6% (-5.6% ppts.| -100.0%).\n",
|
||||
"AVG WAIVER REVENUE PER GJ (not significant): 5.64 vs. 4.91 (0.73 ppts.| 14.9%).\n",
|
||||
"AVG DEPOSIT REVENUE PER GJ (not significant): 0.26 vs. 0.37 (-0.11 ppts.| -28.9%).\n",
|
||||
"AVG CIH REVENUE PER GJ (not significant): 0.0 vs. 0.52 (-0.52 ppts.| -100.0%).\n",
|
||||
"AVG CSAT PER GJ WITH RESPONSE (not significant): 3.0 vs. 4.0 (-1.0 ppts.| -25.0%).\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -665,9 +685,9 @@
|
|||
" stat_sign = row[1]['is_significant_95']\n",
|
||||
"\n",
|
||||
" if stat_sign:\n",
|
||||
" print(f\"{metric} - SIGNIFICANT RESULT: {value_b} vs. {value_a} ({abs_diff} | {rel_diff}).\")\n",
|
||||
" print(f\"{metric} - SIGNIFICANT RESULT: {value_b} vs. {value_a} ({abs_diff} ppts.| {rel_diff}).\")\n",
|
||||
" else:\n",
|
||||
" print(f\"{metric} (not significant): {value_b} vs. {value_a} ({abs_diff} | {rel_diff}).\")\n",
|
||||
" print(f\"{metric} (not significant): {value_b} vs. {value_a} ({abs_diff} ppts.| {rel_diff}).\")\n",
|
||||
"\n",
|
||||
"# Print main metrics\n",
|
||||
"print_metrics(main_metrics_rows, header=\"Main Metrics - Comparing {} vs. {}.\".format(var_B, var_A))\n",
|
||||
|
|
@ -675,13 +695,6 @@
|
|||
"# Print other metrics\n",
|
||||
"print_metrics(other_metrics_rows, header=\"Other Metrics\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue