diff --git a/forecast_comp.py b/forecast_comp.py new file mode 100644 index 0000000..2767a17 --- /dev/null +++ b/forecast_comp.py @@ -0,0 +1,170 @@ +""" +Compare two forecasts: one point alone vs with extra points mixed in. + +Run: + python forecast_comp.py + +Prints MAE for both models — does not write JSON or change charts. +""" + +import json +import re +import warnings +from pathlib import Path + +import pandas as pd +from sklearn.linear_model import Ridge +from sklearn.metrics import mean_absolute_error +from skforecast.exceptions import IgnoredArgumentWarning +from skforecast.recursive import ForecasterRecursive + +# harmless when predictions are flat (same value every step) — see skforecast docs +warnings.simplefilter("ignore", category=IgnoredArgumentWarning) + +# --- change these to experiment --- +JS_FILE = "js/rtu-sample.js" #file your reading data from +TARGET = "RaTmp" #point your making the model for +WITH_POINTS = ["Oa"] # will take Target + all points in WITH_POINTS and make a second model out of them +LAGS = 25 + +folder = Path(__file__).parent + + +def load_sample_js(path: str) -> dict: + """Read var data = { ... }; from a .js file.""" + text = (folder / path).read_text(encoding="utf-8") + return json.loads(re.search(r"=\s*(\{.*\})\s*;?\s*$", text, re.DOTALL).group(1)) + + +def to_series(rows: list, name: str) -> pd.Series: + """Turn [{x, y}, ...] into an evenly spaced time series.""" + times = pd.to_datetime([r["x"] for r in rows], unit="ms") + values = [float(r["y"]) for r in rows] + series = pd.Series(values, index=times, name=name).sort_index() + step = series.index.to_series().diff().median() + return series.resample(step).mean().interpolate() + + +def check_data_quality(target, y, extras, y_test, extras_test): + """ + Warn when MAE results may look too good or mean little. + Returns a list of human-readable warning strings. + """ + msgs = [] + + # flat TARGET in the backtest slice (last 20%) + if len(y_test) == 0: + msgs.append(" Test slice is empty — check your data.") #empty set + return msgs + if y_test.nunique() == 1: + msgs.append( + f" TARGET '{target}' is flat in the test slice (always {y_test.iloc[0]:.4g}) " #stale data + "— MAE can be 0 or misleading." + ) + elif y_test.std() < 0.05: + msgs.append( + f" TARGET '{target}' barely moves in the test slice (std={y_test.std():.3f})." #less stale but still stale + ) + + for col in extras.columns: + # duplicate of TARGET — same values at every timestamp + diff = (y - extras[col]).abs() + if diff.max() < 0.01: + msgs.append( + f" '{col}' is identical to TARGET — combined MAE ~0 does not mean a breakthrough." #data is the exact same + ) + else: + corr = y.corr(extras[col]) + if corr is not None and abs(corr) > 0.995: + msgs.append( + f" '{col}' is almost the same as TARGET (corr={corr:.3f})." #data is almost the exact same + ) + + # lookup table — each WITH_POINTS value maps to only one TARGET value + target_levels_per_exog = y.groupby(extras[col]).nunique() + if len(target_levels_per_exog) > 1 and (target_levels_per_exog <= 1).all(): + msgs.append( + f" '{col}' almost fully determines TARGET (each value → one TARGET) " #points are directly related + ) + + # flat extra column in test slice + if extras_test[col].nunique() == 1: + msgs.append( + f" '{col}' is flat in the test slice (always {extras_test[col].iloc[0]:.4g})." + ) + + return msgs + + +def print_quality_warnings(msgs): + if not msgs: + return + print("Warnings (read before trusting MAE):") + for line in msgs: + print(line) + print() + + +# 1) LOAD — target + each extra point from the same .js file +WITH_POINTS = [name.strip() for name in WITH_POINTS if name and str(name).strip()] +if not WITH_POINTS: + raise SystemExit("Set at least one name in WITH_POINTS (e.g. ['FanSts']).") + +data = load_sample_js(JS_FILE) +if TARGET not in data: + raise SystemExit(f"TARGET '{TARGET}' not found in {JS_FILE}") +for name in WITH_POINTS: + if name not in data: + raise SystemExit(f"WITH_POINTS '{name}' not found in {JS_FILE}") + +y = to_series(data[TARGET], TARGET) +extra_series = [to_series(data[name], name) for name in WITH_POINTS] + +# 2) ALIGN — one row per timestamp; drop rows where any column is missing +df = pd.concat([y] + extra_series, axis=1).dropna() +y = df[TARGET] +extras = df[WITH_POINTS] + +print(f"File: {JS_FILE}") +print(f"Target: {TARGET}") +print(f"Also using: {', '.join(WITH_POINTS)}") +print(f"Aligned readings: {len(y)}\n") + +# 3) TRAIN / TEST split — first 80% train, last 20% hidden for backtest +split = int(len(y) * 0.8) +y_train, y_test = y.iloc[:split], y.iloc[split:] +extras_train, extras_test = extras.iloc[:split], extras.iloc[split:] + +quality_msgs = check_data_quality(TARGET, y, extras, y_test, extras_test) +print_quality_warnings(quality_msgs) + +# 4) MODEL A — target only (same as forecast.py for one point) +alone = ForecasterRecursive(estimator=Ridge(), lags=LAGS) +alone.fit(y=y_train) +pred_alone = alone.predict(steps=len(y_test)) +pred_alone.index = y_test.index +mae_alone = mean_absolute_error(y_test, pred_alone) + +# 5) MODEL B — target + extra points at each timestamp +combined = ForecasterRecursive(estimator=Ridge(), lags=LAGS) +combined.fit(y=y_train, exog=extras_train) +pred_combined = combined.predict(steps=len(y_test), exog=extras_test) +pred_combined.index = y_test.index +mae_combined = mean_absolute_error(y_test, pred_combined) + +# 6) COMPARE — lower MAE wins +with_label = ", ".join(WITH_POINTS) +print("Backtest on last 20% of sample data:") +print(f" {TARGET} only: MAE = {mae_alone:.2f}") +print(f" {TARGET} + [{with_label}]: MAE = {mae_combined:.2f}") + +diff = mae_alone - mae_combined +if diff > 0.05: + print(f"\nExtra points helped — about {diff:.2f} better on average.") +elif diff < -0.05: + print(f"\nExtra points did not help here — {abs(diff):.2f} worse.") +else: + print("\nAbout the same on this slice of data.") + +if mae_combined < 0.01 and quality_msgs: + print("\nNote: combined MAE ~0 — re-read warnings above before celebrating.")