171 lines
6.2 KiB
Python
171 lines
6.2 KiB
Python
"""
|
|
Compare two forecasts: one point alone vs with extra points mixed in.
|
|
|
|
Run:
|
|
python forecast_comp.py
|
|
|
|
Prints MAE for both models — does not write JSON or change charts.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
import warnings
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
from sklearn.linear_model import Ridge
|
|
from sklearn.metrics import mean_absolute_error
|
|
from skforecast.exceptions import IgnoredArgumentWarning
|
|
from skforecast.recursive import ForecasterRecursive
|
|
|
|
# harmless when predictions are flat (same value every step) — see skforecast docs
|
|
warnings.simplefilter("ignore", category=IgnoredArgumentWarning)
|
|
|
|
# --- change these to experiment ---
|
|
JS_FILE = "js/rtu-sample.js" #file your reading data from
|
|
TARGET = "RaTmp" #point your making the model for
|
|
WITH_POINTS = ["Oa"] # will take Target + all points in WITH_POINTS and make a second model out of them
|
|
LAGS = 25
|
|
|
|
folder = Path(__file__).parent
|
|
|
|
|
|
def load_sample_js(path: str) -> dict:
|
|
"""Read var data = { ... }; from a .js file."""
|
|
text = (folder / path).read_text(encoding="utf-8")
|
|
return json.loads(re.search(r"=\s*(\{.*\})\s*;?\s*$", text, re.DOTALL).group(1))
|
|
|
|
|
|
def to_series(rows: list, name: str) -> pd.Series:
|
|
"""Turn [{x, y}, ...] into an evenly spaced time series."""
|
|
times = pd.to_datetime([r["x"] for r in rows], unit="ms")
|
|
values = [float(r["y"]) for r in rows]
|
|
series = pd.Series(values, index=times, name=name).sort_index()
|
|
step = series.index.to_series().diff().median()
|
|
return series.resample(step).mean().interpolate()
|
|
|
|
|
|
def check_data_quality(target, y, extras, y_test, extras_test):
|
|
"""
|
|
Warn when MAE results may look too good or mean little.
|
|
Returns a list of human-readable warning strings.
|
|
"""
|
|
msgs = []
|
|
|
|
# flat TARGET in the backtest slice (last 20%)
|
|
if len(y_test) == 0:
|
|
msgs.append(" Test slice is empty — check your data.") #empty set
|
|
return msgs
|
|
if y_test.nunique() == 1:
|
|
msgs.append(
|
|
f" TARGET '{target}' is flat in the test slice (always {y_test.iloc[0]:.4g}) " #stale data
|
|
"— MAE can be 0 or misleading."
|
|
)
|
|
elif y_test.std() < 0.05:
|
|
msgs.append(
|
|
f" TARGET '{target}' barely moves in the test slice (std={y_test.std():.3f})." #less stale but still stale
|
|
)
|
|
|
|
for col in extras.columns:
|
|
# duplicate of TARGET — same values at every timestamp
|
|
diff = (y - extras[col]).abs()
|
|
if diff.max() < 0.01:
|
|
msgs.append(
|
|
f" '{col}' is identical to TARGET — combined MAE ~0 does not mean a breakthrough." #data is the exact same
|
|
)
|
|
else:
|
|
corr = y.corr(extras[col])
|
|
if corr is not None and abs(corr) > 0.995:
|
|
msgs.append(
|
|
f" '{col}' is almost the same as TARGET (corr={corr:.3f})." #data is almost the exact same
|
|
)
|
|
|
|
# lookup table — each WITH_POINTS value maps to only one TARGET value
|
|
target_levels_per_exog = y.groupby(extras[col]).nunique()
|
|
if len(target_levels_per_exog) > 1 and (target_levels_per_exog <= 1).all():
|
|
msgs.append(
|
|
f" '{col}' almost fully determines TARGET (each value → one TARGET) " #points are directly related
|
|
)
|
|
|
|
# flat extra column in test slice
|
|
if extras_test[col].nunique() == 1:
|
|
msgs.append(
|
|
f" '{col}' is flat in the test slice (always {extras_test[col].iloc[0]:.4g})."
|
|
)
|
|
|
|
return msgs
|
|
|
|
|
|
def print_quality_warnings(msgs):
|
|
if not msgs:
|
|
return
|
|
print("Warnings (read before trusting MAE):")
|
|
for line in msgs:
|
|
print(line)
|
|
print()
|
|
|
|
|
|
# 1) LOAD — target + each extra point from the same .js file
|
|
WITH_POINTS = [name.strip() for name in WITH_POINTS if name and str(name).strip()]
|
|
if not WITH_POINTS:
|
|
raise SystemExit("Set at least one name in WITH_POINTS (e.g. ['FanSts']).")
|
|
|
|
data = load_sample_js(JS_FILE)
|
|
if TARGET not in data:
|
|
raise SystemExit(f"TARGET '{TARGET}' not found in {JS_FILE}")
|
|
for name in WITH_POINTS:
|
|
if name not in data:
|
|
raise SystemExit(f"WITH_POINTS '{name}' not found in {JS_FILE}")
|
|
|
|
y = to_series(data[TARGET], TARGET)
|
|
extra_series = [to_series(data[name], name) for name in WITH_POINTS]
|
|
|
|
# 2) ALIGN — one row per timestamp; drop rows where any column is missing
|
|
df = pd.concat([y] + extra_series, axis=1).dropna()
|
|
y = df[TARGET]
|
|
extras = df[WITH_POINTS]
|
|
|
|
print(f"File: {JS_FILE}")
|
|
print(f"Target: {TARGET}")
|
|
print(f"Also using: {', '.join(WITH_POINTS)}")
|
|
print(f"Aligned readings: {len(y)}\n")
|
|
|
|
# 3) TRAIN / TEST split — first 80% train, last 20% hidden for backtest
|
|
split = int(len(y) * 0.8)
|
|
y_train, y_test = y.iloc[:split], y.iloc[split:]
|
|
extras_train, extras_test = extras.iloc[:split], extras.iloc[split:]
|
|
|
|
quality_msgs = check_data_quality(TARGET, y, extras, y_test, extras_test)
|
|
print_quality_warnings(quality_msgs)
|
|
|
|
# 4) MODEL A — target only (same as forecast.py for one point)
|
|
alone = ForecasterRecursive(estimator=Ridge(), lags=LAGS)
|
|
alone.fit(y=y_train)
|
|
pred_alone = alone.predict(steps=len(y_test))
|
|
pred_alone.index = y_test.index
|
|
mae_alone = mean_absolute_error(y_test, pred_alone)
|
|
|
|
# 5) MODEL B — target + extra points at each timestamp
|
|
combined = ForecasterRecursive(estimator=Ridge(), lags=LAGS)
|
|
combined.fit(y=y_train, exog=extras_train)
|
|
pred_combined = combined.predict(steps=len(y_test), exog=extras_test)
|
|
pred_combined.index = y_test.index
|
|
mae_combined = mean_absolute_error(y_test, pred_combined)
|
|
|
|
# 6) COMPARE — lower MAE wins
|
|
with_label = ", ".join(WITH_POINTS)
|
|
print("Backtest on last 20% of sample data:")
|
|
print(f" {TARGET} only: MAE = {mae_alone:.2f}")
|
|
print(f" {TARGET} + [{with_label}]: MAE = {mae_combined:.2f}")
|
|
|
|
diff = mae_alone - mae_combined
|
|
if diff > 0.05:
|
|
print(f"\nExtra points helped — about {diff:.2f} better on average.")
|
|
elif diff < -0.05:
|
|
print(f"\nExtra points did not help here — {abs(diff):.2f} worse.")
|
|
else:
|
|
print("\nAbout the same on this slice of data.")
|
|
|
|
if mae_combined < 0.01 and quality_msgs:
|
|
print("\nNote: combined MAE ~0 — re-read warnings above before celebrating.")
|