After training N-BEATS model, I want to predict with an out of sample dataframe with the length of max_encoder_lentgh as an input, and should have an output of max_prediction_length.
It throws "AssertionError: filters should not remove entries all entries - check encoder/decoder lengths and lags" if the length of the dataframe is lower than: max_encoder_length + max_prediction_length.
This makes impossible to forecast in out-of-sample data.
gist: https://gist.github.com/fornasari12/5ae6dc414f730beb2bd5c8e267b6c9f1 env:
pytorch-forecasting==0.9.0
pytorch-lightning==1.5.0
lightning-flash[tabular]>=0.5.2
torch==1.8.1
statsmodels==0.12.1
sktime==0.8.1
scikit-learn==0.24.2
pyarrow==3.0.0
pandas==1.2.5
numpy==1.20.3
matplotlib==3.4.3
PyYAML==6.0
import os
import warnings
from config import load_config
from load_data import LoadData
import pandas as pd
import pytorch_lightning as pl
from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
from pytorch_lightning.loggers import TensorBoardLogger
from pytorch_lightning.callbacks import EarlyStopping
import torch
from pytorch_forecasting import Baseline, NBeats, TimeSeriesDataSet
from pytorch_forecasting.data import NaNLabelEncoder
from pytorch_forecasting.data import GroupNormalizer
from pytorch_forecasting.data.examples import generate_ar_data
from pytorch_forecasting.metrics import SMAPE
warnings.filterwarnings("ignore")
if __name__ == "__main__":
timesteps = 1000
data = generate_ar_data(seasonality=10.0, timesteps=timesteps, n_series=100, seed=42)
data["static"] = 2
data["date"] = pd.Timestamp("2020-01-01") + pd.to_timedelta(data.time_idx, "D")
data.series = data.series.astype(str).astype("category")
max_encoder_length = 30
max_prediction_length = 15
cutoff = timesteps * 0.70
train_data = data[data["time_idx"] <= cutoff]
test_data = data[data["time_idx"] > cutoff]
training = TimeSeriesDataSet(
train_data,
time_idx="time_idx",
target="value",
# categorical_encoders={"series": NaNLabelEncoder().fit(train_data.series)},
group_ids=["series"],
time_varying_unknown_reals=["value"],
max_encoder_length=max_encoder_length,
max_prediction_length=max_prediction_length,
# allow_missing_timesteps=True,
)
# training_cutoff = train_data["time_idx"].max() - max_prediction_length
validation = TimeSeriesDataSet.from_dataset(training, train_data, predict=True, stop_randomization=True)
batch_size = 1024
train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0)
val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size * 10, num_workers=0)
# configure network and trainer
pl.seed_everything(42)
early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=10, verbose=False, mode="min")
trainer = pl.Trainer(
max_epochs=3,
gpus=0,
weights_summary="top",
gradient_clip_val=0.01,
callbacks=[early_stop_callback],
# limit_train_batches=30,
)
net = NBeats.from_dataset(
training,
learning_rate=0.01,
log_interval=10,
log_val_interval=1,
weight_decay=1e-2,
widths=[32, 512],
backcast_loss_ratio=1.0,
)
trainer.fit(
net,
train_dataloader=train_dataloader,
val_dataloaders=val_dataloader,
)
# THIS PRODUCES:
# AssertionError: filters should not remove entries all entries - check encoder/decoder lengths and lags
y_hat_tft = net.predict(
test_data[test_data["series"] == '0'][:max_encoder_length],
mode="prediction",
return_x=True)