1# Copyright 2017-2020 Spotify AB
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15from abc import ABC, abstractmethod
16from typing import Union, Iterable, Tuple, List
17
18from pandas import DataFrame
19
20from spotify_confidence.chartgrid import ChartGrid
21from .confidence_computer_abc import ConfidenceComputerABC
22from .confidence_grapher_abc import ConfidenceGrapherABC
23from ..constants import NIM_TYPE
24
25
26class ConfidenceABC(ABC):
27 @property
28 def _confidence_computer(self) -> ConfidenceComputerABC:
29 return self._computer
30
31 @_confidence_computer.setter
32 def _confidence_computer(self, computer: ConfidenceComputerABC):
33 self._computer = computer
34
35 @property
36 def _confidence_grapher(self) -> ConfidenceGrapherABC:
37 return self._grapher
38
39 @_confidence_grapher.setter
40 def _confidence_grapher(self, grapher: ConfidenceGrapherABC):
41 self._grapher = grapher
42
43 @abstractmethod
44 def __init__(
45 self,
46 data_frame: DataFrame,
47 numerator_column: str,
48 numerator_sum_squares_column: Union[str, None],
49 denominator_column: str,
50 categorical_group_columns: Union[str, Iterable, None],
51 ordinal_group_column: Union[str, None],
52 interval_size: float,
53 correction_method: str,
54 metric_column: Union[str, None],
55 treatment_column: Union[str, None],
56 power: float,
57 ):
58 pass
59
60 @abstractmethod
61 def summary(self, verbose: bool) -> DataFrame:
62 """Args:
63 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe.
64 Returns:
65 Dataframe containing summary statistics
66 """
67 pass
68
69 @abstractmethod
70 def difference(
71 self,
72 level_1: Union[str, Tuple],
73 level_2: Union[str, Tuple],
74 absolute: bool,
75 groupby: Union[str, Iterable],
76 non_inferiority_margins: NIM_TYPE,
77 final_expected_sample_size_column: str,
78 verbose: bool,
79 minimum_detectable_effects_column: str,
80 ) -> DataFrame:
81 """Args:
82 groupby (str): Name of column.
83 If specified, will plot a separate chart for each level of the
84 grouping.
85 non_inferiority_margins (Union[Tuple[float, str], Dict[str, Tuple[float, str]], bool]):
86 Pass tuple(non_inferiority_margin, preferred direction) to use the same NIM for all
87 comparisons, e.g. (0.01, 'increase'), which means that we want
88 level_2 to be grater than the average of level_1 times (1-0.01),
89 or (0.05, 'decrease') which means that we want
90 level_2 to be smaller than the average
91 of level_1 times (1+0.01).
92 Pass dictionary {{group:tuple(non_inferiority_margin, preferred direction}} to use
93 different non-inferiority margins for different values of
94 groupby column.
95 To performe a one-sided test without nim, use
96 (None, preffered direction).
97 Alternatively, pass True to use the "non_inferiority_margin" and "preferred_direction"
98 columns of dataframe that was passed to the contructor, as source of nims.
99 final_expected_sample_size_column (str): Column in source data frame containing expected number of
100 observations at end of experiment.
101 Use in combination with ordinal groupby to perform a
102 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.
103 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe.
104 minimum_detectable_effects_column (str): The minimum detectable effect, used for calculating required
105 sample size.
106
107 Returns:
108 Dataframe containing the difference in means between
109 group 1 and 2, p-values and confidence intervals for each value
110 in the groupby column
111 """
112 pass
113
114 @abstractmethod
115 def differences(
116 self,
117 levels: List[Tuple],
118 absolute: bool,
119 groupby: Union[str, Iterable],
120 non_inferiority_margins: NIM_TYPE,
121 final_expected_sample_size_column: str,
122 verbose: bool,
123 minimum_detectable_effects_column: str,
124 ) -> DataFrame:
125 """Args:
126 levels: (list(tuple)): list of levels to compare
127 groupby (str): Name of column.
128 If specified, will plot a separate chart for each level of the
129 grouping.
130 non_inferiority_margins (Union[Tuple[float, str], Dict[str, Tuple[float, str]], bool]):
131 Pass tuple(non_inferiority_margin, preferred direction) to use the same NIM for all
132 comparisons, e.g. (0.01, 'increase'), which means that we want
133 level_2 to be grater than the average of level_1 times (1-0.01),
134 or (0.05, 'decrease') which means that we want
135 level_2 to be smaller than the average
136 of level_1 times (1+0.01).
137 Pass dictionary {{group:tuple(non_inferiority_margin, preferred direction}} to use
138 different non-inferiority margins for different values of
139 groupby column.
140 To performe a one-sided test without nim, use
141 (None, preffered direction).
142 Alternatively, pass True to use the "non_inferiority_margin" and "preferred_direction"
143 columns of dataframe that was passed to the contructor, as source of nims.
144 final_expected_sample_size_column (str): Column in source data frame containing expected number of
145 observations at end of experiment.
146 Use in combination with ordinal groupby to perform a
147 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.
148 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe.
149 minimum_detectable_effects_column (str): The minimum detectable effect, used for calculating required
150 sample size.
151
152 Returns:
153 Dataframe containing the difference in means between
154 group 1 and 2, p-values and confidence intervals for each value
155 in the groupby column
156 """
157 pass
158
159 @abstractmethod
160 def multiple_difference(
161 self,
162 level: Union[str, Tuple],
163 absolute: bool,
164 groupby: Union[str, Iterable],
165 level_as_reference: bool,
166 non_inferiority_margins: NIM_TYPE,
167 final_expected_sample_size_column: str,
168 verbose: bool,
169 minimum_detectable_effects_column: str,
170 ) -> DataFrame:
171 """Args:
172 groupby (str): Name of column.
173 If specified, will plot a separate chart for each level of the
174 grouping.
175 level_as_reference (bool):
176 If false (default), compare level to all other
177 groups. If true, compare all other groups to level.
178 non_inferiority_margins (Union[Tuple[float, str],
179 Dict[str, Tuple[float, str]]]):
180 Pass tuple(nim, preferred direction) to use the same NIM for all
181 comparisons, e.g. (0.01, 'increase'), which means that we want
182 level_2 to be grater than the average of level_1 times (1-0.01),
183 or (0.05, 'decrease') which means that we want
184 level_2 to be smaller than the average
185 of level_1 times (1+0.01).
186 Pass dictionary {{group:tuple(nim, preferred direction}} to use
187 different non-inferiority margins for different values of
188 groupby column.
189 To performe a one-sided test without nim, use
190 (None, preffered direction).
191 Alternatively, pass True to use the "non_inferiority_margin" and "preferred_direction"
192 columns of dataframe that was passed to the contructor, as source of nims.
193 final_expected_sample_size_column (str): Column in source data frame containing expected number of
194 observations at end of experiment.
195 Use in combination with ordinal groupby to perform a
196 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.
197 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe.
198 minimum_detectable_effects_column (str): The minimum detectable effect, used for calculating required
199 sample size.
200
201 Returns:
202 Dataframe containing the difference in means between
203 group 1 and 2, p-values and confidence intervals for each value
204 in the groupby column
205 """
206 pass
207
208 @abstractmethod
209 def summary_plot(self, groupby: Union[str, Iterable]) -> ChartGrid:
210 """Plot for each group in the data_frame:
211
212 if ordinal level exists:
213 line graph with area to represent confidence interval
214 if categorical levels:
215 Interval plots of confidence intervals by group
216
217 Args:
218 groupby (str): Name of column.
219 If specified, will plot a separate chart for each level of the
220 grouping.
221
222 Returns:
223 ChartGrid object and a DataFrame with numerical results.
224 """
225 pass
226
227 @abstractmethod
228 def difference_plot(
229 self,
230 level_1: Union[str, Tuple],
231 level_2: Union[str, Tuple],
232 absolute: bool,
233 groupby: Union[str, Iterable],
234 non_inferiority_margins: NIM_TYPE,
235 use_adjusted_intervals: bool,
236 final_expected_sample_size_column: str,
237 ) -> ChartGrid:
238 """Plot representing the difference between group 1 and 2.
239 - Difference in means or proportions, depending
240 on the response variable type.
241
242 - Plot interval plot with confidence interval of the
243 difference between groups
244
245 Args:
246 level_1 (str, tuple of str): Name of first level.
247 level_2 (str, tuple of str): Name of second level.
248 absolute (bool): If True then return the absolute
249 difference (level2 - level1)
250 otherwise return the relative difference (level2 / level1 - 1)
251 groupby (str): Name of column, or list of columns.
252 If specified, will return an interval for each level
253 of the grouped dimension, or a confidence band if the
254 grouped dimension is ordinal
255 non_inferiority_margins (Union[Tuple[float, str],
256 Dict[str, Tuple[float, str]]]):
257 Pass tuple(nim, preferred direction) to use the same NIM for all
258 comparisons, e.g. (0.01, 'increase'), which means that we want
259 level_2 to be grater than the average of level_1 times (1-0.01),
260 or (0.05, 'decrease') which means that we want
261 level_2 to be smaller than the average
262 of level_1 times (1+0.01).
263 Pass dictionary {{group:tuple(nim, preferred direction}} to use
264 different non-inferiority margins for different values of
265 groupby column.
266 To performe a one-sided test without nim, use
267 (None, preffered direction).
268 use_adjusted_intervals (bool):
269 If true, use e.g. bon-ferroni corrected
270 (or other method provided) confidence intervals
271 final_expected_sample_size_column (str): Column in source data frame containing expected number of
272 observations at end of experiment.
273 Use in combination with ordinal groupby to perform a
274 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.
275
276 Returns:
277 Chartify Chart object and a DataFrame with numerical results.
278 """
279
280 @abstractmethod
281 def differences_plot(
282 self,
283 levels: List[Tuple],
284 absolute: bool,
285 groupby: Union[str, Iterable],
286 non_inferiority_margins: NIM_TYPE,
287 use_adjusted_intervals: bool,
288 final_expected_sample_size_column: str,
289 ) -> ChartGrid:
290 """Plot representing the difference between group 1 and 2.
291 - Difference in means or proportions, depending
292 on the response variable type.
293
294 - Plot interval plot with confidence interval of the
295 difference between groups
296
297 Args:
298 levels: (list(tuple)): list of levels to compare
299 absolute (bool): If True then return the absolute
300 difference (level2 - level1)
301 otherwise return the relative difference (level2 / level1 - 1)
302 groupby (str): Name of column, or list of columns.
303 If specified, will return an interval for each level
304 of the grouped dimension, or a confidence band if the
305 grouped dimension is ordinal
306 non_inferiority_margins (Union[Tuple[float, str],
307 Dict[str, Tuple[float, str]]]):
308 Pass tuple(nim, preferred direction) to use the same NIM for all
309 comparisons, e.g. (0.01, 'increase'), which means that we want
310 level_2 to be grater than the average of level_1 times (1-0.01),
311 or (0.05, 'decrease') which means that we want
312 level_2 to be smaller than the average
313 of level_1 times (1+0.01).
314 Pass dictionary {{group:tuple(nim, preferred direction}} to use
315 different non-inferiority margins for different values of
316 groupby column.
317 To performe a one-sided test without nim, use
318 (None, preffered direction).
319 use_adjusted_intervals (bool):
320 If true, use e.g. bon-ferroni corrected
321 (or other method provided) confidence intervals
322 final_expected_sample_size_column (str): Column in source data frame containing expected number of
323 observations at end of experiment.
324 Use in combination with ordinal groupby to perform a
325 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.
326
327 Returns:
328 Chartify Chart object and a DataFrame with numerical results.
329 """
330
331 @abstractmethod
332 def multiple_difference_plot(
333 self,
334 level: Union[str, Tuple],
335 absolute: bool,
336 groupby: Union[str, Iterable],
337 level_as_reference: bool,
338 non_inferiority_margins: NIM_TYPE,
339 use_adjusted_intervals: bool,
340 final_expected_sample_size_column: str,
341 ) -> ChartGrid:
342 """Compare level to all other groups or, if level_as_reference = True,
343 all other groups to level.
344
345 Args:
346 level (str, tuple of str): Name of level.
347 absolute (bool): If True then return the absolute
348 difference (level2 - level1)
349 otherwise return the relative difference (level2 / level1 - 1)
350 groupby (str): Name of column, or list of columns.
351 If specified, will return an interval for each level
352 of the grouped dimension, or a confidence band if the
353 grouped dimension is ordinal
354 level_as_reference: If false (default), compare level to all other
355 groups. If true, compare all other groups to level.
356 non_inferiority_margins (Union[Tuple[float, str],
357 Dict[str, Tuple[float, str]]]):
358 Pass tuple(nim, preferred direction) to use the same NIM for all
359 comparisons, e.g. (0.01, 'increase'), which means that we want
360 level_2 to be grater than the average of level_1 times (1-0.01),
361 or (0.05, 'decrease') which means that we want
362 level_2 to be smaller than the average
363 of level_1 times (1+0.01).
364 Pass dictionary {{group:tuple(nim, preferred direction}} to use
365 different non-inferiority margins for different values of
366 groupby column.
367 To performe a one-sided test without nim, use
368 (None, preffered direction).
369 use_adjusted_intervals (bool):
370 If true, use e.g. bon-ferroni corrected
371 (or other method provided) confidence intervals
372 final_expected_sample_size_column (str): Column in source data frame containing expected number of
373 observations at end of experiment.
374 Use in combination with ordinal groupby to perform a
375 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details.
376
377 Returns:
378 ChartGrid object and a DataFrame with numerical results.
379 """