"""Reference values for Scintria's core statistics (pack 130).

Creates core.csv from a fixed seed and writes ref-core.json with the values SciPy and statsmodels give for the same
data. Scintria's own results for this data are computed when its /validation page is built and compared with these.
Run:  python core_reference.py   (SciPy 1.18, statsmodels 0.15, NumPy 2.5, pandas 3.0 were used.)
"""
import json
import numpy as np
import pandas as pd
import scipy
import scipy.stats as st
import statsmodels
import statsmodels.api as sm
import statsmodels.formula.api as smf
from statsmodels.stats.contingency_tables import StratifiedTable, Table2x2
from statsmodels.stats.power import NormalIndPower
from statsmodels.stats.proportion import samplesize_proportions_2indep_onetail

rng = np.random.default_rng(130)
n = 180
arm = np.array(["A"] * 70 + ["B"] * 60 + ["C"] * 50)
rng.shuffle(arm)
sex = np.where(rng.random(n) < 0.45, "M", "F")
age = np.round(rng.normal(41, 12, n).clip(18, 80), 1)
sd_by_arm = {"A": 9.0, "B": 13.0, "C": 17.0}  # unequal variances
sbp = np.round(104 + 0.45 * age + np.where(arm == "B", 4.0, 0) - np.where(arm == "C", 3.0, 0) + np.where(sex == "M", 3.5, 0)
               + np.array([rng.normal(0, sd_by_arm[a]) for a in arm]), 1)
lp = -1.2 + 0.03 * (age - 41) + np.where(arm == "B", 0.7, 0) + np.where(arm == "C", 1.1, 0) - np.where(sex == "M", 0.4, 0)
cured = (rng.random(n) < 1 / (1 + np.exp(-lp))).astype(int)
months = np.round(rng.uniform(6, 24, n), 1)
rate = np.exp(-2.3 + np.where(arm == "B", 0.35, 0) + np.where(arm == "C", -0.25, 0) + 0.01 * (age - 41))
visits = rng.poisson(rate * months)
score = np.round(np.exp(rng.normal(1.6, 0.6, n)) + np.where(sex == "M", 0.8, 0) + np.where(arm == "C", 1.0, 0))  # skewed, heavy ties
site = rng.choice(["North", "East", "West"], n, p=[0.4, 0.35, 0.25])

df = pd.DataFrame({"id": np.arange(1, n + 1), "arm": arm, "sex": sex, "age": age, "sbp": sbp, "cured": cured,
                   "visits": visits, "months": months, "score": score.astype(int), "site": site})
df.to_csv("core.csv", index=False)

R = {"software": {"scipy": scipy.__version__, "statsmodels": statsmodels.__version__, "numpy": np.__version__, "pandas": pd.__version__}}
F_, M_ = df[df.sex == "F"], df[df.sex == "M"]

w = st.ttest_ind(F_.sbp, M_.sbp, equal_var=False)
ci = w.confidence_interval(0.95)
R["welch"] = {"t": w.statistic, "df": w.df, "p": w.pvalue, "low": ci.low, "high": ci.high}
mw = st.mannwhitneyu(F_.score, M_.score, use_continuity=True, alternative="two-sided", method="asymptotic")
R["mannwhitney"] = {"U": mw.statistic, "p": mw.pvalue}
groups = [df[df.arm == a].sbp for a in ["A", "B", "C"]]
an = st.f_oneway(*groups)
R["anova"] = {"F": an.statistic, "p": an.pvalue}
kw = st.kruskal(*[df[df.arm == a].score for a in ["A", "B", "C"]])
R["kruskal"] = {"H": kw.statistic, "p": kw.pvalue}
tab = pd.crosstab(df.arm, df.cured).loc[["A", "B", "C"], [1, 0]].values
chi = st.chi2_contingency(tab, correction=False)
R["chisq"] = {"table": tab.tolist(), "chi2": chi.statistic, "df": int(chi.dof), "p": chi.pvalue}
t22 = pd.crosstab(df.sex, df.cured).loc[["M", "F"], [1, 0]].values  # exposed = M, event = cured
R["fisher"] = {"table": t22.tolist(), "p": st.fisher_exact(t22).pvalue}
T = Table2x2(t22)
R["or_woolf"] = {"or": T.oddsratio, "low": T.oddsratio_confint()[0], "high": T.oddsratio_confint()[1]}
R["rr_katz"] = {"rr": T.riskratio, "low": T.riskratio_confint()[0], "high": T.riskratio_confint()[1]}
pr = st.pearsonr(df.age, df.sbp)
pci = pr.confidence_interval(0.95)
R["pearson"] = {"r": pr.statistic, "p": pr.pvalue, "low": pci.low, "high": pci.high}
sp = st.spearmanr(df.age, df.score)
R["spearman"] = {"rho": sp.statistic, "p": sp.pvalue}

def coefs(res, names):
    return {"names": names, "coef": [float(res.params[k]) for k in names], "se": [float(res.bse[k]) for k in names],
            "p": [float(res.pvalues[k]) for k in names]}

names = ["Intercept", "age", "arm[T.B]", "arm[T.C]", "sex[T.M]"]
ols = smf.ols("sbp ~ age + arm + sex", df).fit()
R["linear"] = {**coefs(ols, names), "r2": ols.rsquared, "adjr2": ols.rsquared_adj, "F": ols.fvalue, "llf": ols.llf}
logit = smf.logit("cured ~ age + arm + sex", df).fit(disp=0, tol=1e-12, maxiter=100)
R["logistic"] = {**coefs(logit, names), "llf": logit.llf, "llnull": logit.llnull}
pois = smf.glm("visits ~ arm + age", df, family=sm.families.Poisson(), offset=np.log(df.months)).fit(tol=1e-12)
R["poisson"] = {**coefs(pois, ["Intercept", "arm[T.B]", "arm[T.C]", "age"]), "llf": pois.llf}

strata = []
for s in ["North", "East", "West"]:
    d = df[df.site == s]
    strata.append(pd.crosstab(d.sex, d.cured).reindex(index=["M", "F"], columns=[1, 0], fill_value=0).values)
S = StratifiedTable(strata)
bd = S.test_equal_odds(adjust=False)
R["mh"] = {"strata": [x.tolist() for x in strata], "or": S.oddsratio_pooled, "low": S.oddsratio_pooled_confint()[0],
           "high": S.oddsratio_pooled_confint()[1], "rr": S.riskratio_pooled, "bd_chi2": bd.statistic, "bd_p": bd.pvalue}

ss = []
for diff, sd, k in [(5, 12, 1), (5, 12, 2), (0.4, 1.0, 1)]:
    n1 = NormalIndPower().solve_power(effect_size=diff / sd, alpha=0.05, power=0.8, ratio=k, alternative="two-sided")
    ss.append({"diff": diff, "sd": sd, "ratio": k, "n1": n1})
R["ss_means"] = ss
sp2 = []
for p1, p2, k in [(0.30, 0.45, 1), (0.30, 0.45, 2), (0.10, 0.05, 1)]:
    # statsmodels: sample 1 has prop2 + diff and nobs1; sample 2 has prop2 and ratio × nobs1 (Scintria: group 1 = p1, group 2 = k × n1).
    n1 = samplesize_proportions_2indep_onetail(diff=p1 - p2, prop2=p2, power=0.8, ratio=k, alpha=0.05, alternative="two-sided")
    sp2.append({"p1": p1, "p2": p2, "ratio": k, "n1": n1})
R["ss_props"] = sp2

def clean(o):
    if isinstance(o, dict): return {k: clean(v) for k, v in o.items()}
    if isinstance(o, (list, tuple)): return [clean(v) for v in o]
    if isinstance(o, (np.floating,)): return float(o)
    if isinstance(o, (np.integer,)): return int(o)
    return o

with open("ref-core.json", "w") as f:
    json.dump(clean(R), f, indent=1)
print("wrote core.csv and ref-core.json")
