Coverage for /Users/sebastiana/Documents/Sugarpills/confidence/spotify_confidence/analysis/abstract_base_classes/confidence_abc.py: 84%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

44 statements  

1# Copyright 2017-2020 Spotify AB 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14 

15from abc import ABC, abstractmethod 

16from typing import Union, Iterable, Tuple, List 

17 

18from pandas import DataFrame 

19 

20from spotify_confidence.chartgrid import ChartGrid 

21from .confidence_computer_abc import ConfidenceComputerABC 

22from .confidence_grapher_abc import ConfidenceGrapherABC 

23from ..constants import NIM_TYPE 

24 

25 

26class ConfidenceABC(ABC): 

27 @property 

28 def _confidence_computer(self) -> ConfidenceComputerABC: 

29 return self._computer 

30 

31 @_confidence_computer.setter 

32 def _confidence_computer(self, computer: ConfidenceComputerABC): 

33 self._computer = computer 

34 

35 @property 

36 def _confidence_grapher(self) -> ConfidenceGrapherABC: 

37 return self._grapher 

38 

39 @_confidence_grapher.setter 

40 def _confidence_grapher(self, grapher: ConfidenceGrapherABC): 

41 self._grapher = grapher 

42 

43 @abstractmethod 

44 def __init__( 

45 self, 

46 data_frame: DataFrame, 

47 numerator_column: str, 

48 numerator_sum_squares_column: Union[str, None], 

49 denominator_column: str, 

50 categorical_group_columns: Union[str, Iterable, None], 

51 ordinal_group_column: Union[str, None], 

52 interval_size: float, 

53 correction_method: str, 

54 metric_column: Union[str, None], 

55 treatment_column: Union[str, None], 

56 power: float, 

57 ): 

58 pass 

59 

60 @abstractmethod 

61 def summary(self, verbose: bool) -> DataFrame: 

62 """Args: 

63 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe. 

64 Returns: 

65 Dataframe containing summary statistics 

66 """ 

67 pass 

68 

69 @abstractmethod 

70 def difference( 

71 self, 

72 level_1: Union[str, Tuple], 

73 level_2: Union[str, Tuple], 

74 absolute: bool, 

75 groupby: Union[str, Iterable], 

76 non_inferiority_margins: NIM_TYPE, 

77 final_expected_sample_size_column: str, 

78 verbose: bool, 

79 minimum_detectable_effects_column: str, 

80 ) -> DataFrame: 

81 """Args: 

82 groupby (str): Name of column. 

83 If specified, will plot a separate chart for each level of the 

84 grouping. 

85 non_inferiority_margins (Union[Tuple[float, str], Dict[str, Tuple[float, str]], bool]): 

86 Pass tuple(non_inferiority_margin, preferred direction) to use the same NIM for all 

87 comparisons, e.g. (0.01, 'increase'), which means that we want 

88 level_2 to be grater than the average of level_1 times (1-0.01), 

89 or (0.05, 'decrease') which means that we want 

90 level_2 to be smaller than the average 

91 of level_1 times (1+0.01). 

92 Pass dictionary {{group:tuple(non_inferiority_margin, preferred direction}} to use 

93 different non-inferiority margins for different values of 

94 groupby column. 

95 To performe a one-sided test without nim, use 

96 (None, preffered direction). 

97 Alternatively, pass True to use the "non_inferiority_margin" and "preferred_direction" 

98 columns of dataframe that was passed to the contructor, as source of nims. 

99 final_expected_sample_size_column (str): Column in source data frame containing expected number of 

100 observations at end of experiment. 

101 Use in combination with ordinal groupby to perform a 

102 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details. 

103 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe. 

104 minimum_detectable_effects_column (str): The minimum detectable effect, used for calculating required 

105 sample size. 

106 

107 Returns: 

108 Dataframe containing the difference in means between 

109 group 1 and 2, p-values and confidence intervals for each value 

110 in the groupby column 

111 """ 

112 pass 

113 

114 @abstractmethod 

115 def differences( 

116 self, 

117 levels: List[Tuple], 

118 absolute: bool, 

119 groupby: Union[str, Iterable], 

120 non_inferiority_margins: NIM_TYPE, 

121 final_expected_sample_size_column: str, 

122 verbose: bool, 

123 minimum_detectable_effects_column: str, 

124 ) -> DataFrame: 

125 """Args: 

126 levels: (list(tuple)): list of levels to compare 

127 groupby (str): Name of column. 

128 If specified, will plot a separate chart for each level of the 

129 grouping. 

130 non_inferiority_margins (Union[Tuple[float, str], Dict[str, Tuple[float, str]], bool]): 

131 Pass tuple(non_inferiority_margin, preferred direction) to use the same NIM for all 

132 comparisons, e.g. (0.01, 'increase'), which means that we want 

133 level_2 to be grater than the average of level_1 times (1-0.01), 

134 or (0.05, 'decrease') which means that we want 

135 level_2 to be smaller than the average 

136 of level_1 times (1+0.01). 

137 Pass dictionary {{group:tuple(non_inferiority_margin, preferred direction}} to use 

138 different non-inferiority margins for different values of 

139 groupby column. 

140 To performe a one-sided test without nim, use 

141 (None, preffered direction). 

142 Alternatively, pass True to use the "non_inferiority_margin" and "preferred_direction" 

143 columns of dataframe that was passed to the contructor, as source of nims. 

144 final_expected_sample_size_column (str): Column in source data frame containing expected number of 

145 observations at end of experiment. 

146 Use in combination with ordinal groupby to perform a 

147 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details. 

148 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe. 

149 minimum_detectable_effects_column (str): The minimum detectable effect, used for calculating required 

150 sample size. 

151 

152 Returns: 

153 Dataframe containing the difference in means between 

154 group 1 and 2, p-values and confidence intervals for each value 

155 in the groupby column 

156 """ 

157 pass 

158 

159 @abstractmethod 

160 def multiple_difference( 

161 self, 

162 level: Union[str, Tuple], 

163 absolute: bool, 

164 groupby: Union[str, Iterable], 

165 level_as_reference: bool, 

166 non_inferiority_margins: NIM_TYPE, 

167 final_expected_sample_size_column: str, 

168 verbose: bool, 

169 minimum_detectable_effects_column: str, 

170 ) -> DataFrame: 

171 """Args: 

172 groupby (str): Name of column. 

173 If specified, will plot a separate chart for each level of the 

174 grouping. 

175 level_as_reference (bool): 

176 If false (default), compare level to all other 

177 groups. If true, compare all other groups to level. 

178 non_inferiority_margins (Union[Tuple[float, str], 

179 Dict[str, Tuple[float, str]]]): 

180 Pass tuple(nim, preferred direction) to use the same NIM for all 

181 comparisons, e.g. (0.01, 'increase'), which means that we want 

182 level_2 to be grater than the average of level_1 times (1-0.01), 

183 or (0.05, 'decrease') which means that we want 

184 level_2 to be smaller than the average 

185 of level_1 times (1+0.01). 

186 Pass dictionary {{group:tuple(nim, preferred direction}} to use 

187 different non-inferiority margins for different values of 

188 groupby column. 

189 To performe a one-sided test without nim, use 

190 (None, preffered direction). 

191 Alternatively, pass True to use the "non_inferiority_margin" and "preferred_direction" 

192 columns of dataframe that was passed to the contructor, as source of nims. 

193 final_expected_sample_size_column (str): Column in source data frame containing expected number of 

194 observations at end of experiment. 

195 Use in combination with ordinal groupby to perform a 

196 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details. 

197 verbose (bool): include columns used in intermediate steps in the calculations in returned dataframe. 

198 minimum_detectable_effects_column (str): The minimum detectable effect, used for calculating required 

199 sample size. 

200 

201 Returns: 

202 Dataframe containing the difference in means between 

203 group 1 and 2, p-values and confidence intervals for each value 

204 in the groupby column 

205 """ 

206 pass 

207 

208 @abstractmethod 

209 def summary_plot(self, groupby: Union[str, Iterable]) -> ChartGrid: 

210 """Plot for each group in the data_frame: 

211 

212 if ordinal level exists: 

213 line graph with area to represent confidence interval 

214 if categorical levels: 

215 Interval plots of confidence intervals by group 

216 

217 Args: 

218 groupby (str): Name of column. 

219 If specified, will plot a separate chart for each level of the 

220 grouping. 

221 

222 Returns: 

223 ChartGrid object and a DataFrame with numerical results. 

224 """ 

225 pass 

226 

227 @abstractmethod 

228 def difference_plot( 

229 self, 

230 level_1: Union[str, Tuple], 

231 level_2: Union[str, Tuple], 

232 absolute: bool, 

233 groupby: Union[str, Iterable], 

234 non_inferiority_margins: NIM_TYPE, 

235 use_adjusted_intervals: bool, 

236 final_expected_sample_size_column: str, 

237 ) -> ChartGrid: 

238 """Plot representing the difference between group 1 and 2. 

239 - Difference in means or proportions, depending 

240 on the response variable type. 

241 

242 - Plot interval plot with confidence interval of the 

243 difference between groups 

244 

245 Args: 

246 level_1 (str, tuple of str): Name of first level. 

247 level_2 (str, tuple of str): Name of second level. 

248 absolute (bool): If True then return the absolute 

249 difference (level2 - level1) 

250 otherwise return the relative difference (level2 / level1 - 1) 

251 groupby (str): Name of column, or list of columns. 

252 If specified, will return an interval for each level 

253 of the grouped dimension, or a confidence band if the 

254 grouped dimension is ordinal 

255 non_inferiority_margins (Union[Tuple[float, str], 

256 Dict[str, Tuple[float, str]]]): 

257 Pass tuple(nim, preferred direction) to use the same NIM for all 

258 comparisons, e.g. (0.01, 'increase'), which means that we want 

259 level_2 to be grater than the average of level_1 times (1-0.01), 

260 or (0.05, 'decrease') which means that we want 

261 level_2 to be smaller than the average 

262 of level_1 times (1+0.01). 

263 Pass dictionary {{group:tuple(nim, preferred direction}} to use 

264 different non-inferiority margins for different values of 

265 groupby column. 

266 To performe a one-sided test without nim, use 

267 (None, preffered direction). 

268 use_adjusted_intervals (bool): 

269 If true, use e.g. bon-ferroni corrected 

270 (or other method provided) confidence intervals 

271 final_expected_sample_size_column (str): Column in source data frame containing expected number of 

272 observations at end of experiment. 

273 Use in combination with ordinal groupby to perform a 

274 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details. 

275 

276 Returns: 

277 Chartify Chart object and a DataFrame with numerical results. 

278 """ 

279 

280 @abstractmethod 

281 def differences_plot( 

282 self, 

283 levels: List[Tuple], 

284 absolute: bool, 

285 groupby: Union[str, Iterable], 

286 non_inferiority_margins: NIM_TYPE, 

287 use_adjusted_intervals: bool, 

288 final_expected_sample_size_column: str, 

289 ) -> ChartGrid: 

290 """Plot representing the difference between group 1 and 2. 

291 - Difference in means or proportions, depending 

292 on the response variable type. 

293 

294 - Plot interval plot with confidence interval of the 

295 difference between groups 

296 

297 Args: 

298 levels: (list(tuple)): list of levels to compare 

299 absolute (bool): If True then return the absolute 

300 difference (level2 - level1) 

301 otherwise return the relative difference (level2 / level1 - 1) 

302 groupby (str): Name of column, or list of columns. 

303 If specified, will return an interval for each level 

304 of the grouped dimension, or a confidence band if the 

305 grouped dimension is ordinal 

306 non_inferiority_margins (Union[Tuple[float, str], 

307 Dict[str, Tuple[float, str]]]): 

308 Pass tuple(nim, preferred direction) to use the same NIM for all 

309 comparisons, e.g. (0.01, 'increase'), which means that we want 

310 level_2 to be grater than the average of level_1 times (1-0.01), 

311 or (0.05, 'decrease') which means that we want 

312 level_2 to be smaller than the average 

313 of level_1 times (1+0.01). 

314 Pass dictionary {{group:tuple(nim, preferred direction}} to use 

315 different non-inferiority margins for different values of 

316 groupby column. 

317 To performe a one-sided test without nim, use 

318 (None, preffered direction). 

319 use_adjusted_intervals (bool): 

320 If true, use e.g. bon-ferroni corrected 

321 (or other method provided) confidence intervals 

322 final_expected_sample_size_column (str): Column in source data frame containing expected number of 

323 observations at end of experiment. 

324 Use in combination with ordinal groupby to perform a 

325 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details. 

326 

327 Returns: 

328 Chartify Chart object and a DataFrame with numerical results. 

329 """ 

330 

331 @abstractmethod 

332 def multiple_difference_plot( 

333 self, 

334 level: Union[str, Tuple], 

335 absolute: bool, 

336 groupby: Union[str, Iterable], 

337 level_as_reference: bool, 

338 non_inferiority_margins: NIM_TYPE, 

339 use_adjusted_intervals: bool, 

340 final_expected_sample_size_column: str, 

341 ) -> ChartGrid: 

342 """Compare level to all other groups or, if level_as_reference = True, 

343 all other groups to level. 

344 

345 Args: 

346 level (str, tuple of str): Name of level. 

347 absolute (bool): If True then return the absolute 

348 difference (level2 - level1) 

349 otherwise return the relative difference (level2 / level1 - 1) 

350 groupby (str): Name of column, or list of columns. 

351 If specified, will return an interval for each level 

352 of the grouped dimension, or a confidence band if the 

353 grouped dimension is ordinal 

354 level_as_reference: If false (default), compare level to all other 

355 groups. If true, compare all other groups to level. 

356 non_inferiority_margins (Union[Tuple[float, str], 

357 Dict[str, Tuple[float, str]]]): 

358 Pass tuple(nim, preferred direction) to use the same NIM for all 

359 comparisons, e.g. (0.01, 'increase'), which means that we want 

360 level_2 to be grater than the average of level_1 times (1-0.01), 

361 or (0.05, 'decrease') which means that we want 

362 level_2 to be smaller than the average 

363 of level_1 times (1+0.01). 

364 Pass dictionary {{group:tuple(nim, preferred direction}} to use 

365 different non-inferiority margins for different values of 

366 groupby column. 

367 To performe a one-sided test without nim, use 

368 (None, preffered direction). 

369 use_adjusted_intervals (bool): 

370 If true, use e.g. bon-ferroni corrected 

371 (or other method provided) confidence intervals 

372 final_expected_sample_size_column (str): Column in source data frame containing expected number of 

373 observations at end of experiment. 

374 Use in combination with ordinal groupby to perform a 

375 sequential test. See https://cran.r-project.org/web/packages/ldbounds/index.html for details. 

376 

377 Returns: 

378 ChartGrid object and a DataFrame with numerical results. 

379 """