Coverage for /Users/mschultzberg/PycharmProjects/confidence/spotify_confidence/analysis/frequentist/confidence_computers/z_test_computer.py : 71%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
STD_ERR, PREFERENCE_TEST, NULL_HYPOTHESIS, DIFFERENCE, ALPHA, IS_SIGNIFICANT, HOLM, SPOT_1_HOLM, HOMMEL, \ SIMES_HOCHBERG, SPOT_1_HOMMEL, SPOT_1_SIMES_HOCHBERG, NIM, ADJUSTED_ALPHA, ADJUSTED_POWER, MDE,\ ALTERNATIVE_HYPOTHESIS
return bounds(t, alpha, rho=2, ztrun=8, sides=sides, max_nints=1000)
raise ValueError('''Can't compute point estimate: denominator is 0''')
row[self._numerator_sumsq] / row[self._denominator] - row[POINT_ESTIMATE] ** 2) raise ValueError('Computed variance is negative. ' 'Please check your inputs.')
row[VARIANCE + SFX2] / row[self._denominator + SFX2])
mean=row[POINT_ESTIMATE], std_mean=np.sqrt(row[VARIANCE] / row[self._denominator]), alpha=1-self._interval_size, alternative=TWO_SIDED )
value2=row[POINT_ESTIMATE + SFX1], std_diff=row[STD_ERR], alternative=row[PREFERENCE_TEST], diff=row[NULL_HYPOTHESIS])
mean=row[DIFFERENCE], std_mean=row[STD_ERR], alpha=row[alpha_column], alternative=row[PREFERENCE_TEST])
df: DataFrame, mde: float, alpha: float) -> DataFrame: v1, v2 = df[VARIANCE + SFX1], df[VARIANCE + SFX2] n1, n2 = df[self._denominator + SFX1], df[self._denominator + SFX2]
var_pooled = ((n1 - 1) * v1 + (n2 - 1) * v2) / (n1 + n2 - 2)
return power_calculation(mde, var_pooled, alpha, n1, n2)
df: DataFrame, final_expected_sample_size_column: str, filtered_sufficient_statistics: DataFrame, num_comparisons: int): total_sample_size = ( filtered_sufficient_statistics.groupby(df.index.names) .agg({self._denominator: sum, final_expected_sample_size_column: np.mean}) .rename(columns={self._denominator: f'total_{self._denominator}'}) ) groups_except_ordinal = [ column for column in df.index.names if column != self._ordinal_group_column] max_sample_size_by_group = ( total_sample_size[f'total_{self._denominator}'].max() if len(groups_except_ordinal) == 0 else total_sample_size.groupby(groups_except_ordinal)[f'total_{self._denominator}'].max())
if type(max_sample_size_by_group) is not Series: total_sample_size = total_sample_size.assign(**{f'total_{self._denominator}_max': max_sample_size_by_group}) else: total_sample_size = total_sample_size.merge(right=max_sample_size_by_group, left_index=True, right_index=True, suffixes=('', '_max'))
total_sample_size = ( total_sample_size .assign(final_expected_sample_size=lambda df: df[[f'total_{self._denominator}_max', final_expected_sample_size_column]].max(axis=1)) .assign( sample_size_proportions=lambda df: df['total_' + self._denominator]/df['final_expected_sample_size'] ) )
def adjusted_alphas_for_group(grp: DataFrame) -> Series: return ( sequential_bounds( t=grp['sample_size_proportions'].values, alpha=grp[ALPHA].values[0]/num_comparisons, sides=2 if (grp[PREFERENCE_TEST] == TWO_SIDED).all() else 1 ).df .set_index(grp.index) .assign(adjusted_alpha=lambda df: df.apply( lambda row: 2 * (1 - st.norm.cdf(row['zb'])) if (grp[PREFERENCE_TEST] == TWO_SIDED).all() else 1 - st.norm.cdf(row['zb']), axis=1)) )[['zb', 'adjusted_alpha']]
return ( df.merge(total_sample_size, left_index=True, right_index=True) .groupby(groups_except_ordinal + ['level_1', 'level_2'])[['sample_size_proportions', PREFERENCE_TEST, ALPHA]] .apply(adjusted_alphas_for_group) .reset_index().set_index(df.index.names) )['adjusted_alpha']
self, df: DataFrame, correction_method: str, alpha: float, w: float = 1.0, ) -> Tuple[Union[Series, float], Union[Series, float]]: raise ValueError( "CIs can only be produced for one-sided tests when other multiple test corrections " "methods than bonferroni are applied" )
else: raise ValueError("CIs not supported for correction method. " f"Supported methods: {HOMMEL}, {HOLM}, {SIMES_HOCHBERG}," f"{SPOT_1_HOLM}, {SPOT_1_HOMMEL} and {SPOT_1_SIMES_HOCHBERG}")
alpha_adj = adjusted_alpha_rej_equal_m alpha_adj = adjusted_alpha_rej_less_m else:
else: bound2 = min(row[NULL_HYPOTHESIS], bound1)
df: Series, kappa: float, proportion_of_total: float, z_alpha: float, z_power: float, binary: bool, current_number_of_units: int, non_inferiority: bool ) -> Series:
control_avg=df[POINT_ESTIMATE + SFX1], control_var=df[VARIANCE + SFX1], non_inferiority=False, kappa=kappa, proportion_of_total=proportion_of_total, current_number_of_units=current_number_of_units, z_alpha=z_alpha, z_power=z_power, )[0] else: binary_metric=binary, non_inferiority=df[NIM] is not None, control_avg=df[POINT_ESTIMATE + SFX1], control_var=df[VARIANCE + SFX1], hypothetical_effect=0 ) df[VARIANCE + SFX1] / kappa + treatment_var) n2_partial + kappa * n2_partial))
binary: Union[Series, bool], non_inferiority: Union[Series, bool], hypothetical_effect: Union[Series, float], control_avg: Union[Series, float], control_var: Union[Series, float], z_alpha: float = None, kappa: float = None, proportion_of_total: Union[Series, float] = None, z_power: float = None, ) -> Union[Series, float]:
raise ValueError('kappa is None, must be postive float') raise ValueError('proportion_of_total is None, must be between 0 and 1')
binary, non_inferiority, control_avg, control_var, hypothetical_effect )
z_alpha=z_alpha, z_power=z_power, hypothetical_effect=hypothetical_effect, control_var=control_var, treatment_var=treatment_var, kappa=kappa, )
df: Series, ) -> Series:
non_inferiority = not np.isnan(df[NIM]) else: raise ValueError('NIM has to be type float or None.')
kappa=kappa, proportion_of_total=proportion_of_total, z_alpha=z_alpha, z_power=z_power, binary=binary, current_number_of_units=current_number_of_units, non_inferiority=non_inferiority)
z_alpha=z_alpha, z_power=z_power, binary=binary, non_inferiority=non_inferiority, hypothetical_effect = df[ALTERNATIVE_HYPOTHESIS + SFX1] - df[NULL_HYPOTHESIS], control_avg=df[POINT_ESTIMATE + SFX1], control_var=df[VARIANCE + SFX1], kappa=kappa ) else:
control_avg: float, control_var: float, binary_metric: bool, non_inferiority: bool = False, power: float = None, alpha: float = None, kappa: float = None, proportion_of_total: float = None, current_number_of_units: float = None, ): z_alpha = st.norm.ppf(1 - alpha) z_power = st.norm.ppf(power)
if binary_metric and not non_inferiority: effect = self._search_MDE_binary_local_search( control_avg=control_avg, control_var=control_var, non_inferiority=non_inferiority, kappa=kappa, proportion_of_total=proportion_of_total, current_number_of_units=current_number_of_units, z_alpha=z_alpha, z_power=z_power, )[0] else: treatment_var = self._get_hypothetical_treatment_var( binary_metric, non_inferiority, control_avg, control_var, hypothetical_effect=0 ) n2_partial = np.power((z_alpha + z_power), 2) * ( control_var / kappa + treatment_var) effect = np.sqrt((1 / (current_number_of_units * proportion_of_total)) * ( n2_partial + kappa * n2_partial))
return effect
control_avg: float, control_var: float, non_inferiority: bool, kappa: float, proportion_of_total: float, current_number_of_units: float, z_alpha: float = None, z_power: float = None, ): hypothetical_effect=x, control_avg=control_avg, control_var=control_var, binary=True, non_inferiority=non_inferiority, kappa=kappa, proportion_of_total=proportion_of_total, current_number_of_units=current_number_of_units, z_alpha=z_alpha, z_power=z_power, )
# corner case that crashes the optimizer return min_val, f(min_val)
# we stop immediately if a solution was found that is "good enough". A threshold of # 1 indicates that # the approximated number of units (based on the current effect candidate) is off by # at most 1.0
# take next value from queue
# conduct a bounded local search, using a very small tol value improved # performance during tests # result = optimize.minimize_scalar(f, bounds=(interval[0], interval[1]), # method='bounded', tol=10e-14) f, bounds=(interval[0], interval[1]), method="bounded", options={"xatol": 10e-14, "maxiter": 50} )
# add new bounds to the queue
else: # check if grid search finds a better solution alt_result_x, alt_result_fun = self.search_MDE_binary( control_avg, control_var, non_inferiority, kappa, proportion_of_total, current_number_of_units, z_alpha, z_power, return_cost_val=True, )
return (alt_result_x, alt_result_fun) if alt_result_fun < best_fun else ( best_x, best_fun)
control_avg: float, control_var: float, non_inferiority: bool, kappa: float, proportion_of_total: float, current_number_of_units: float, z_alpha: float = None, z_power: float = None, return_cost_val=False, ): candidate_effects = np.linspace(10e-9, 1 - control_avg, num=2000) for i in range(2): test = [] for effect in candidate_effects: test.append( self.find_current_powered_effect( hypothetical_effect=effect, control_avg=control_avg, control_var=control_var, binary=True, non_inferiority=non_inferiority, kappa=kappa, proportion_of_total=proportion_of_total, current_number_of_units=current_number_of_units, z_alpha=z_alpha, z_power=z_power, ) )
test = np.array(test) index = [idx for idx, element in enumerate(test) if element == test.min()] if len(index) != 1: index = [index[int(np.ceil(len(index) / 2))]] if i == 0: if index[0] == 9999: return np.inf lower_effect_bound = 10e-9 if index[0] == 0 else candidate_effects[ index[0] - 1] candidate_effects = np.linspace(lower_effect_bound, candidate_effects[index[0]], num=10000)
index = [idx for idx, element in enumerate(test) if element == test.min()]
return candidate_effects[index[0]], test[index[0]] if return_cost_val else \ candidate_effects[index[0]]
z_alpha: float, z_power: float, hypothetical_effect: float, control_var: float, treatment_var: float, kappa: float, ) -> float: control_var / kappa + treatment_var))
hypothetical_effect: float, control_avg: float, control_var: float, binary: bool, non_inferiority: bool, kappa: float, proportion_of_total: float, current_number_of_units: float, z_power: float = None, z_alpha: float = None, ) -> float:
binary_metric=binary, non_inferiority=non_inferiority, control_avg=control_avg, control_var=control_var, hypothetical_effect=hypothetical_effect ) control_var, treatment_var, kappa, )
current_number_of_units - ((n2 + n2 * kappa) / proportion_of_total), 2)
binary_metric: bool, non_inferiority: bool, control_avg: float, control_var: float, hypothetical_effect: float, ) -> float: # For binary metrics, the variance can be derived from the average. However, # we do *not* do this for # non-inferiority tests because for non-inferiority tests, the basic assumption # is that the # mean of the control group and treatment group are identical. else:
|