Ejemplo regularización

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import warnings  # Para ignorar mensajes de advertencia

warnings.filterwarnings("ignore")

Descargar datos desde Yahoo Finance:

import yfinance as yf

tickers = ["BTC-USD"]
ohlc = yf.download(tickers, period="max")
print(ohlc.tail())

[*******************100%*********************]  1 of 1 completed
                    Open          High           Low         Close  Date
2022-09-03  19969.718750  20037.009766  19698.355469  19832.087891
2022-09-04  19832.470703  19999.689453  19636.816406  19986.712891
2022-09-05  19988.789062  20031.160156  19673.046875  19812.371094
2022-09-06  19817.724609  20155.269531  18800.171875  18837.667969
2022-09-08  19309.482422  19372.404297  19275.042969  19276.115234

               Adj Close       Volume
Date
2022-09-03  19832.087891  23613051457
2022-09-04  19986.712891  25245861652
2022-09-05  19812.371094  28813460025
2022-09-06  18837.667969  43403978910
2022-09-08  19276.115234  34544746496

df = ohlc["Adj Close"].dropna(how="all")
df.tail()

Date
2022-09-03    19832.087891
2022-09-04    19986.712891
2022-09-05    19812.371094
2022-09-06    18837.667969
2022-09-08    19276.115234
Name: Adj Close, dtype: float64

df.shape

(2913,)

df = np.array(df[:, np.newaxis])
df.shape

(2913, 1)

plt.figure(figsize=(10, 6))
plt.plot(df);

Conjunto de train y test:

time_test = 200
train = df[:len(df)-time_test]
test = df[len(df)-time_test:]

Función para conformar el dataset para datos secuenciales:

[time_step, Features]

def split_sequence(sequence, time_step):
    X, y = list(), list()
    for i in range(len(sequence)):
        end_ix = i + time_step
        if end_ix > len(sequence) - 1:
            break
        seq_x, seq_y = sequence[i:end_ix], sequence[end_ix]
        X.append(seq_x)
        y.append(seq_y)
    return np.array(X), np.array(y)

time_step = 3

X_train, y_train = split_sequence(train, time_step)
X_test, y_test = split_sequence(test, time_step)

Ajuste del modelo inicial:

from keras.models import Sequential
from keras.layers import Dense
import keras

model = Sequential()
model.add(Dense(200, activation="relu", input_shape=(time_step,)))
model.add(Dense(200, activation="relu"))
model.add(Dense(200, activation="relu"))
model.add(Dense(1))
model.compile(optimizer="adam", loss="mse")
history = model.fit(
    X_train,
    y_train,
    validation_data=(X_test, y_test),
    epochs=200,
    batch_size=50,
    verbose=0
)

# Evaluación del desempeño:
rmse = model.evaluate(X_test, y_test, verbose=0) ** 0.5
print(rmse)
plt.plot(range(1, len(history.epoch) + 1), history.history["loss"], label="Train")
plt.plot(range(1, len(history.epoch) + 1), history.history["val_loss"], label="Test")
plt.xlabel("epoch")
plt.ylabel("Loss")
plt.legend();

1118.935934269697

Early Stopping:

En el siguiente ejemplo el modelo se agrega una parada anticipada (early stopping) si el modelo no mejora durante 20 epochs. En caso de no usar este método, la red de entrenará por 200 epoch y podrá observar que el modelo converge desde las primeras epochs.

Este método es útil para detener el entrenamiento y no esperar hasta que ejecute la última epoch que indiquemos.

keras.backend.clear_session()

model = Sequential()
model.add(Dense(200, activation="relu", input_shape=(time_step,)))
model.add(Dense(200, activation="relu"))
model.add(Dense(200, activation="relu"))
model.add(Dense(1))
model.compile(optimizer="adam", loss="mse")

callbacks= keras.callbacks.EarlyStopping(   # Interrumpe el entrenamiento cuando se detiene la mejora.
    monitor="val_loss",                 # Supervisa el accuracy en la validación del modelo.
    patience=20,        # Interrumpe el entrenamiento cuando la precisión ha dejado de mejorar durante 10 epochs.
    restore_best_weights=True)

history = model.fit(
    X_train,
    y_train,
    validation_data=(X_test, y_test),
    callbacks=callbacks,             # Early stopping
    epochs=100,
    batch_size=50,
    verbose=0
)

# Evaluación del desempeño:
rmse = model.evaluate(X_test, y_test, verbose=0) ** 0.5
print(rmse)
plt.plot(range(1, len(history.epoch) + 1), history.history["loss"], label="Train")
plt.plot(range(1, len(history.epoch) + 1), history.history["val_loss"], label="Test")
plt.xlabel("epoch")
plt.ylabel("Loss")
plt.legend();

1124.7939811361011

Regularización L1:

keras.backend.clear_session()

model = Sequential()
model.add(Dense(200, activation="relu", input_shape=(time_step,),
               kernel_regularizer='l1'))                          # Regularización L1 primera capa oculta
model.add(Dense(200, activation="relu", kernel_regularizer='l1'))  # Regularización L1 en segunda capa oculta
model.add(Dense(200, activation="relu", kernel_regularizer='l1'))  # Regularización L1 en tercera capa oculta
model.add(Dense(1))
model.compile(optimizer="adam", loss="mse")
history = model.fit(
    X_train,
    y_train,
    validation_data=(X_test, y_test),
    epochs=100,
    batch_size=50,
    verbose=0
)

# Evaluación del desempeño:
rmse = model.evaluate(X_test, y_test, verbose=0) ** 0.5
print(rmse)
plt.plot(range(1, len(history.epoch) + 1), history.history["loss"], label="Train")
plt.plot(range(1, len(history.epoch) + 1), history.history["val_loss"], label="Test")
plt.xlabel("epoch")
plt.ylabel("Loss")
plt.legend();

1122.4477159315707

Regularización L2:

keras.backend.clear_session()

model = Sequential()
model.add(Dense(200, activation="relu", input_shape=(time_step,),
               kernel_regularizer='l2'))                           # Regularización L2 primera capa oculta
model.add(Dense(200, activation="relu", kernel_regularizer='l2'))  # Regularización L2 en segunda capa oculta
model.add(Dense(200, activation="relu", kernel_regularizer='l2'))  # Regularización L2 en tercera capa oculta
model.add(Dense(1))
model.compile(optimizer="adam", loss="mse")
history = model.fit(
    X_train,
    y_train,
    validation_data=(X_test, y_test),
    epochs=100,
    batch_size=50,
    verbose=0
)

# Evaluación del desempeño:
rmse = model.evaluate(X_test, y_test, verbose=0) ** 0.5
print(rmse)
plt.plot(range(1, len(history.epoch) + 1), history.history["loss"], label="Train")
plt.plot(range(1, len(history.epoch) + 1), history.history["val_loss"], label="Test")
plt.xlabel("epoch")
plt.ylabel("Loss")
plt.legend();

1155.8512014961095

Regularización L2 con Early Stopping:

keras.backend.clear_session()

model = Sequential()
model.add(Dense(200, activation="relu", input_shape=(time_step,),
               kernel_regularizer='l2'))         # Regularización L2 primera capa oculta
model.add(Dense(200, activation="relu", kernel_regularizer='l2'))  # Regularización L2 en segunda capa oculta
model.add(Dense(200, activation="relu", kernel_regularizer='l2'))  # Regularización L2 en tercera capa oculta
model.add(Dense(1))
model.compile(optimizer="adam", loss="mse")

callbacks= keras.callbacks.EarlyStopping(
    monitor="val_loss",
    patience=20,
    restore_best_weights=True)


history = model.fit(
    X_train,
    y_train,
    validation_data=(X_test, y_test),
    callbacks=callbacks,
    epochs=500,                                  # 500 epochs
    batch_size=50,
    verbose=0
)

# Evaluación del desempeño:
rmse = model.evaluate(X_test, y_test, verbose=0) ** 0.5
print(rmse)
plt.plot(range(1, len(history.epoch) + 1), history.history["loss"], label="Train")
plt.plot(range(1, len(history.epoch) + 1), history.history["val_loss"], label="Test")
plt.xlabel("epoch")
plt.ylabel("Loss")
plt.legend();

1123.5660861738397

Dropout:

from keras.layers import Dropout

keras.backend.clear_session()

model = Sequential()
model.add(Dense(200, activation="relu", input_shape=(time_step,)))
model.add(Dropout(0.2))              # Regularización Dropout en primera capa oculta
model.add(Dense(200, activation="relu"))
model.add(Dropout(0.2))              # Regularización Dropout en segunda capa oculta
model.add(Dense(200, activation="relu"))
model.add(Dropout(0.2))              # Regularización Dropout en tercera capa oculta
model.add(Dense(1))
model.compile(optimizer="adam", loss="mse")
history = model.fit(
    X_train,
    y_train,
    validation_data=(X_test, y_test),
    epochs=100,                                  # 100 epochs
    batch_size=50,
    verbose=0
)

# Evaluación del desempeño:
rmse = model.evaluate(X_test, y_test, verbose=0) ** 0.5
print(rmse)
plt.plot(range(1, len(history.epoch) + 1), history.history["loss"], label="Train")
plt.plot(range(1, len(history.epoch) + 1), history.history["val_loss"], label="Test")
plt.xlabel("epoch")
plt.ylabel("Loss")
plt.legend();

3861.8776521272653

Regularización Dropout con Early Stopping:

keras.backend.clear_session()

model = Sequential()
model.add(Dense(200, activation="relu", ))
model.add(Dropout(0.2))
model.add(Dense(200, activation="relu", ))
model.add(Dropout(0.2))
model.add(Dense(200, activation="relu", ))
model.add(Dropout(0.2))
model.add(Dense(1))
model.compile(optimizer="adam", loss="mse")

callbacks= keras.callbacks.EarlyStopping(
    monitor="val_loss",
    patience=20,
    restore_best_weights=True)

history = model.fit(
    X_train,
    y_train,
    validation_data=(X_test, y_test),
    callbacks=callbacks,
    epochs=500,                                  # 500 epochs
    batch_size=50,
    verbose=0
)

# Evaluación del desempeño:
rmse = model.evaluate(X_test, y_test, verbose=0) ** 0.5
print(rmse)
plt.plot(range(1, len(history.epoch) + 1), history.history["loss"], label="Train")
plt.plot(range(1, len(history.epoch) + 1), history.history["val_loss"], label="Test")
plt.xlabel("epoch")
plt.ylabel("Loss")
plt.legend();

1552.5757630466862

Dropout capa de entrada:

La capa de entrada también puede tener Dropout:

model = Sequential()
model.add(Dropout(0.1, input_shape=(time_step,)))  # Capa de entrada con Droput
model.add(Dense(200, activation="relu", ))         # Primera capa oculta
model.add(Dropout(0.2))                            # Dropout primera capa oculta