from getdist.preprocessing import parse_tb, resample_df, drop_zero_df, get_hour_df, find_cron_hours, get_days, create_crons, get_interval_crons,get_hours_from_cron
from getdist.dl_models import LSTMAE
from getdist.plot import PrsPlots
import importlib
import getdist as gd
import pandas as pd
import matplotlib.pyplot as plt
from croniter import croniter
from datetime import datetime
from croniter import croniter_range
import warnings
warnings.filterwarnings('ignore')
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = 'all'
importlib.reload(gd.preprocessing)
importlib.reload(gd.dl_models)
importlib.reload(gd.plot)
# TODO (Darien)
data = pd.read_csv('data/sample_ze_data.csv')
data2 = pd.read_csv('data/sample_ze_data2.csv')
data3 = pd.read_csv('data/sample_ze_data3.csv')
data4 = pd.read_csv('data/sample_ze_data4.csv')
data5 = pd.read_csv('data/sample_ze_data5.csv')
data6 = pd.read_csv('data/sample_ze_data6.csv')
data7 = pd.read_csv('data/sample_ze_data7.csv')
IESO_data = pd.read_csv('data/table_IESO.csv', index_col= False)
GEOGRAIN_data = pd.read_csv('data/GEOGRAIN_data.csv')
PJM_G_Data = pd.read_csv('data/PJM_G.csv', index_col=False)
PJM_X_Data = pd.read_csv('data/PJM_X.csv', index_col=False)
MISO_data = pd.read_csv('data/MISO.csv', index_col=False)
PJM_A_data = pd.read_csv('data/PJM_A.csv', index_col= False)
# data.head(1)
# data2.head(1)
parsed_df = parse_tb(GEOGRAIN_data)
parsed_df.head(3)
2017-12-19T15:02:47.000000 | |
2017-12-19T15:02:47.000000 | |
2017-12-19T15:02:47.000000 | |
| 3 rows × 0 columns | |
IESO_parsed = parse_tb(IESO_data)
resample_df(parsed_df,'H')
| Hint640 – 168 | ||
|---|---|---|
2016-12-12T18:00:00.000000 | 168 | |
2016-12-12T19:00:00.000000 | 0 | |
2016-12-12T20:00:00.000000 | 0 | |
2016-12-12T21:00:00.000000 | 0 | |
2016-12-12T22:00:00.000000 | 0 | |
| Expand rows 5 - 28589 | ||
2020-03-18T00:00:00.000000 | 0 | |
2020-03-18T01:00:00.000000 | 0 | |
2020-03-18T02:00:00.000000 | 0 | |
2020-03-18T03:00:00.000000 | 0 | |
2020-03-18T04:00:00.000000 | 24 | |
| 28595 rows × 1 columns | ||
pplot = PrsPlots()
cron_g = 'S 0 0 17 * * ?'
pplot.plot_with_crons(GEOGRAIN_data, 'H', 5, cron_g, 'GEOGRAIN_CASH_BIDS', True, False)
hours_df_g = find_cron_hours(parsed_df,0, True)
hours_df_g
| COUNTint64 | ||
|---|---|---|
0 | 9437 | |
10 | 46 | |
15 | 165373 | |
16 | 642706 | |
| 4 rows × 1 columns | ||
days_dict_g = get_days(parsed_df, hours_df_g)
days_dict_g
crons_g = create_crons(days_dict_g)
crons_g
all_crons_g = get_interval_crons(crons_g,10)
all_crons_g
pplot.plot_interval_crons(GEOGRAIN_data, all_crons_g)
# TODO (Nadia)
# TODO (Frank)
# Get the aggregated dataframe
parse_geograin = parse_tb(GEOGRAIN_data)
resampled_geograin = resample_df(parse_geograin, 'H')
time_geograin = get_hour_df(drop_zero_df(resampled_geograin))
# Amount of time units (days, hours etc.) to look back on
TIME_STEPS = 10
# Creates and saves the model and scaler transformation
model = LSTMAE()
# Note: this retrains the model, and will take substantial amount of time
lstm_model, scaler = model.create_model(time_geograin, TIME_STEPS)
Epoch 1/1000
3/3 [==============================] - 6s 466ms/step - loss: 0.4017 - val_loss: 0.7400
Epoch 2/1000
3/3 [==============================] - 1s 234ms/step - loss: 0.2789 - val_loss: 0.3135
Epoch 3/1000
3/3 [==============================] - 1s 201ms/step - loss: 0.2836 - val_loss: 0.3020
Epoch 4/1000
3/3 [==============================] - 1s 206ms/step - loss: 0.2778 - val_loss: 0.2833
Epoch 5/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.2359 - val_loss: 0.3502
Epoch 6/1000
3/3 [==============================] - 1s 193ms/step - loss: 0.2400 - val_loss: 0.2414
Epoch 7/1000
3/3 [==============================] - 1s 221ms/step - loss: 0.2411 - val_loss: 0.2522
Epoch 8/1000
3/3 [==============================] - 1s 197ms/step - loss: 0.2208 - val_loss: 0.2582
Epoch 9/1000
3/3 [==============================] - 1s 195ms/step - loss: 0.2311 - val_loss: 0.1611
Epoch 10/1000
3/3 [==============================] - 1s 201ms/step - loss: 0.2190 - val_loss: 0.2940
Epoch 11/1000
3/3 [==============================] - 1s 208ms/step - loss: 0.1985 - val_loss: 0.2410
Epoch 12/1000
3/3 [==============================] - 1s 218ms/step - loss: 0.2094 - val_loss: 0.1292
Epoch 13/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.2118 - val_loss: 0.1306
Epoch 14/1000
3/3 [==============================] - 1s 195ms/step - loss: 0.1936 - val_loss: 0.1592
Epoch 15/1000
3/3 [==============================] - 1s 198ms/step - loss: 0.2160 - val_loss: 0.2235
Epoch 16/1000
3/3 [==============================] - 1s 192ms/step - loss: 0.2258 - val_loss: 0.1344
Epoch 17/1000
3/3 [==============================] - 1s 196ms/step - loss: 0.2196 - val_loss: 0.1444
Epoch 18/1000
3/3 [==============================] - 1s 209ms/step - loss: 0.2226 - val_loss: 0.1399
Epoch 19/1000
3/3 [==============================] - 1s 205ms/step - loss: 0.2107 - val_loss: 0.1210
Epoch 20/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1949 - val_loss: 0.1287
Epoch 21/1000
3/3 [==============================] - 1s 199ms/step - loss: 0.2026 - val_loss: 0.1052
Epoch 22/1000
3/3 [==============================] - 1s 201ms/step - loss: 0.2161 - val_loss: 0.0993
Epoch 23/1000
3/3 [==============================] - 1s 219ms/step - loss: 0.1953 - val_loss: 0.1290
Epoch 24/1000
3/3 [==============================] - 1s 198ms/step - loss: 0.2047 - val_loss: 0.1179
Epoch 25/1000
3/3 [==============================] - 1s 200ms/step - loss: 0.2008 - val_loss: 0.1161
Epoch 26/1000
3/3 [==============================] - 1s 192ms/step - loss: 0.1963 - val_loss: 0.1133
Epoch 27/1000
3/3 [==============================] - 1s 201ms/step - loss: 0.1999 - val_loss: 0.0906
Epoch 28/1000
3/3 [==============================] - 1s 205ms/step - loss: 0.1882 - val_loss: 0.0901
Epoch 29/1000
3/3 [==============================] - 1s 202ms/step - loss: 0.2040 - val_loss: 0.1178
Epoch 30/1000
3/3 [==============================] - 1s 199ms/step - loss: 0.1855 - val_loss: 0.1331
Epoch 31/1000
3/3 [==============================] - 1s 198ms/step - loss: 0.2021 - val_loss: 0.0810
Epoch 32/1000
3/3 [==============================] - 1s 198ms/step - loss: 0.1843 - val_loss: 0.1082
Epoch 33/1000
3/3 [==============================] - 1s 195ms/step - loss: 0.1803 - val_loss: 0.1851
Epoch 34/1000
3/3 [==============================] - 1s 204ms/step - loss: 0.1864 - val_loss: 0.2335
Epoch 35/1000
3/3 [==============================] - 1s 204ms/step - loss: 0.1805 - val_loss: 0.1599
Epoch 36/1000
3/3 [==============================] - 1s 195ms/step - loss: 0.2043 - val_loss: 0.1758
Epoch 37/1000
3/3 [==============================] - 1s 206ms/step - loss: 0.1936 - val_loss: 0.1637
Epoch 38/1000
3/3 [==============================] - 1s 202ms/step - loss: 0.2013 - val_loss: 0.0606
Epoch 39/1000
3/3 [==============================] - 1s 207ms/step - loss: 0.2054 - val_loss: 0.0574
Epoch 40/1000
3/3 [==============================] - 1s 208ms/step - loss: 0.1877 - val_loss: 0.1904
Epoch 41/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.1800 - val_loss: 0.1378
Epoch 42/1000
3/3 [==============================] - 1s 208ms/step - loss: 0.1783 - val_loss: 0.0883
Epoch 43/1000
3/3 [==============================] - 1s 206ms/step - loss: 0.1497 - val_loss: 0.2059
Epoch 44/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1678 - val_loss: 0.2092
Epoch 45/1000
3/3 [==============================] - 1s 188ms/step - loss: 0.1894 - val_loss: 0.1630
Epoch 46/1000
3/3 [==============================] - 1s 203ms/step - loss: 0.1712 - val_loss: 0.0744
Epoch 47/1000
3/3 [==============================] - 1s 191ms/step - loss: 0.1971 - val_loss: 0.1056
Epoch 48/1000
3/3 [==============================] - 1s 195ms/step - loss: 0.1718 - val_loss: 0.1722
Epoch 49/1000
3/3 [==============================] - 1s 201ms/step - loss: 0.1871 - val_loss: 0.0618
Epoch 50/1000
3/3 [==============================] - 1s 198ms/step - loss: 0.1578 - val_loss: 0.1212
Epoch 51/1000
3/3 [==============================] - 1s 204ms/step - loss: 0.1605 - val_loss: 0.1094
Epoch 52/1000
3/3 [==============================] - 1s 205ms/step - loss: 0.2052 - val_loss: 0.0810
Epoch 53/1000
3/3 [==============================] - 1s 207ms/step - loss: 0.1736 - val_loss: 0.0562
Epoch 54/1000
3/3 [==============================] - 1s 191ms/step - loss: 0.1849 - val_loss: 0.1479
Epoch 55/1000
3/3 [==============================] - 1s 187ms/step - loss: 0.1614 - val_loss: 0.1950
Epoch 56/1000
3/3 [==============================] - 1s 209ms/step - loss: 0.1815 - val_loss: 0.0859
Epoch 57/1000
3/3 [==============================] - 1s 208ms/step - loss: 0.1788 - val_loss: 0.0581
Epoch 58/1000
3/3 [==============================] - 1s 205ms/step - loss: 0.1664 - val_loss: 0.0371
Epoch 59/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1817 - val_loss: 0.1817
Epoch 60/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1820 - val_loss: 0.1671
Epoch 61/1000
3/3 [==============================] - 1s 201ms/step - loss: 0.1459 - val_loss: 0.0601
Epoch 62/1000
3/3 [==============================] - 1s 197ms/step - loss: 0.1624 - val_loss: 0.0682
Epoch 63/1000
3/3 [==============================] - 1s 216ms/step - loss: 0.1697 - val_loss: 0.1628
Epoch 64/1000
3/3 [==============================] - 1s 205ms/step - loss: 0.1895 - val_loss: 0.1756
Epoch 65/1000
3/3 [==============================] - 1s 207ms/step - loss: 0.1830 - val_loss: 0.0423
Epoch 66/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.1804 - val_loss: 0.1542
Epoch 67/1000
3/3 [==============================] - 1s 206ms/step - loss: 0.1847 - val_loss: 0.1305
Epoch 68/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1610 - val_loss: 0.0593
Epoch 69/1000
3/3 [==============================] - 1s 194ms/step - loss: 0.1612 - val_loss: 0.0460
Epoch 70/1000
3/3 [==============================] - 1s 198ms/step - loss: 0.1488 - val_loss: 0.1139
Epoch 71/1000
3/3 [==============================] - 1s 204ms/step - loss: 0.1616 - val_loss: 0.2016
Epoch 72/1000
3/3 [==============================] - 1s 204ms/step - loss: 0.1651 - val_loss: 0.1314
Epoch 73/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1611 - val_loss: 0.0467
Epoch 74/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1564 - val_loss: 0.0461
Epoch 75/1000
3/3 [==============================] - 1s 197ms/step - loss: 0.1553 - val_loss: 0.0419
Epoch 76/1000
3/3 [==============================] - 1s 196ms/step - loss: 0.1505 - val_loss: 0.0432
Epoch 77/1000
3/3 [==============================] - 1s 202ms/step - loss: 0.1608 - val_loss: 0.1059
Epoch 78/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.1688 - val_loss: 0.1014
# Applies the trained LSTM model onto the dataframe
anomalies_df_geo = model.apply_model(time_geograin, TIME_STEPS, input_tuple = (lstm_model, scaler))
print(anomalies_df_geo)
#anomalies_df_geo.to_csv('GEO_Anomaly_data.csv')
time loss threshold anomaly
DATE_INSERTED
2017-05-19 16:00:00 0.318223 0.023148 0.020077 True
2017-05-20 16:00:00 0.318223 0.009824 0.020077 False
2017-05-21 16:00:00 0.318223 0.009824 0.020077 False
2017-05-22 16:00:00 0.318223 0.009824 0.020077 False
2017-05-23 16:00:00 0.318223 0.009824 0.020077 False
... ... ... ... ...
2017-12-28 15:00:00 -1.066047 0.013807 0.020077 False
2017-12-29 15:00:00 -1.066047 0.013807 0.020077 False
2017-12-30 15:00:00 -1.066047 0.013807 0.020077 False
2018-01-01 15:00:00 -1.066047 0.013807 0.020077 False
2018-01-02 00:00:00 -21.830092 20.750238 0.020077 True
[184 rows x 4 columns]
latest_start = model.anomaly_schedule(anomalies_df_geo, TIME_STEPS)
print("Suggested time: ", anomalies_df_geo.index[latest_start], " to ", anomalies_df_geo.index[-1])
Suggested time: 2017-11-13 15:00:00 to 2018-01-02 00:00:00
# for PJM_G_Data
# Get the aggregated dataframe
parse_PJMG = parse_tb(PJM_G_Data)
resampled_PJMG = resample_df(parse_PJMG, 'H')
time_PJMG = get_hour_df(drop_zero_df(resampled_PJMG))
# Amount of time units (days, hours etc.) to look back on
TIME_STEPS = 10
# Creates and saves the model and scaler transformation
model = LSTMAE()
# Note: this retrains the model, and will take substantial amount of time
lstm_model_pjmg, scaler_pjmg = model.create_model(time_PJMG, TIME_STEPS)
Epoch 1/1000
1/1 [==============================] - 4s 4s/step - loss: 0.6801 - val_loss: 0.9085
Epoch 2/1000
1/1 [==============================] - 0s 147ms/step - loss: 0.6797 - val_loss: 0.9090
Epoch 3/1000
1/1 [==============================] - 0s 138ms/step - loss: 0.6720 - val_loss: 0.9058
Epoch 4/1000
1/1 [==============================] - 0s 186ms/step - loss: 0.6622 - val_loss: 0.9003
Epoch 5/1000
1/1 [==============================] - 0s 175ms/step - loss: 0.6623 - val_loss: 0.8930
Epoch 6/1000
1/1 [==============================] - 0s 172ms/step - loss: 0.6723 - val_loss: 0.8846
Epoch 7/1000
1/1 [==============================] - 0s 141ms/step - loss: 0.6523 - val_loss: 0.8782
Epoch 8/1000
1/1 [==============================] - 0s 184ms/step - loss: 0.6535 - val_loss: 0.8755
Epoch 9/1000
1/1 [==============================] - 0s 163ms/step - loss: 0.6791 - val_loss: 0.8732
Epoch 10/1000
1/1 [==============================] - 0s 149ms/step - loss: 0.6494 - val_loss: 0.8700
Epoch 11/1000
1/1 [==============================] - 0s 185ms/step - loss: 0.6636 - val_loss: 0.8663
Epoch 12/1000
1/1 [==============================] - 0s 188ms/step - loss: 0.6708 - val_loss: 0.8631
Epoch 13/1000
1/1 [==============================] - 0s 160ms/step - loss: 0.6110 - val_loss: 0.8574
Epoch 14/1000
1/1 [==============================] - 0s 147ms/step - loss: 0.6451 - val_loss: 0.8510
Epoch 15/1000
1/1 [==============================] - 0s 181ms/step - loss: 0.6244 - val_loss: 0.8432
Epoch 16/1000
1/1 [==============================] - 0s 165ms/step - loss: 0.6337 - val_loss: 0.8343
Epoch 17/1000
1/1 [==============================] - 0s 155ms/step - loss: 0.6462 - val_loss: 0.8283
Epoch 18/1000
1/1 [==============================] - 0s 172ms/step - loss: 0.6591 - val_loss: 0.8234
Epoch 19/1000
1/1 [==============================] - 0s 167ms/step - loss: 0.6580 - val_loss: 0.8178
Epoch 20/1000
1/1 [==============================] - 0s 142ms/step - loss: 0.5752 - val_loss: 0.8094
Epoch 21/1000
1/1 [==============================] - 0s 183ms/step - loss: 0.6488 - val_loss: 0.7987
Epoch 22/1000
1/1 [==============================] - 0s 179ms/step - loss: 0.6020 - val_loss: 0.7942
Epoch 23/1000
1/1 [==============================] - 0s 135ms/step - loss: 0.6502 - val_loss: 0.7965
Epoch 24/1000
1/1 [==============================] - 0s 177ms/step - loss: 0.6342 - val_loss: 0.7971
Epoch 25/1000
1/1 [==============================] - 0s 180ms/step - loss: 0.5840 - val_loss: 0.7967
Epoch 26/1000
1/1 [==============================] - 0s 161ms/step - loss: 0.6759 - val_loss: 0.7987
Epoch 27/1000
1/1 [==============================] - 0s 153ms/step - loss: 0.6000 - val_loss: 0.7982
Epoch 28/1000
1/1 [==============================] - 0s 177ms/step - loss: 0.6525 - val_loss: 0.7992
Epoch 29/1000
1/1 [==============================] - 0s 170ms/step - loss: 0.6126 - val_loss: 0.7970
Epoch 30/1000
1/1 [==============================] - 0s 146ms/step - loss: 0.6526 - val_loss: 0.7948
Epoch 31/1000
1/1 [==============================] - 0s 185ms/step - loss: 0.6003 - val_loss: 0.7902
Epoch 32/1000
1/1 [==============================] - 0s 163ms/step - loss: 0.5887 - val_loss: 0.7852
Epoch 33/1000
1/1 [==============================] - 0s 146ms/step - loss: 0.6162 - val_loss: 0.7793
Epoch 34/1000
1/1 [==============================] - 0s 185ms/step - loss: 0.6476 - val_loss: 0.7752
Epoch 35/1000
1/1 [==============================] - 0s 160ms/step - loss: 0.5892 - val_loss: 0.7679
Epoch 36/1000
1/1 [==============================] - 0s 147ms/step - loss: 0.6009 - val_loss: 0.7597
Epoch 37/1000
1/1 [==============================] - 0s 185ms/step - loss: 0.7271 - val_loss: 0.7568
Epoch 38/1000
1/1 [==============================] - 0s 176ms/step - loss: 0.6524 - val_loss: 0.7528
Epoch 39/1000
1/1 [==============================] - 0s 132ms/step - loss: 0.5413 - val_loss: 0.7443
Epoch 40/1000
1/1 [==============================] - 0s 182ms/step - loss: 0.6447 - val_loss: 0.7394
Epoch 41/1000
1/1 [==============================] - 0s 168ms/step - loss: 0.6740 - val_loss: 0.7416
Epoch 42/1000
1/1 [==============================] - 0s 140ms/step - loss: 0.5537 - val_loss: 0.7413
Epoch 43/1000
1/1 [==============================] - 0s 183ms/step - loss: 0.5973 - val_loss: 0.7450
Epoch 44/1000
1/1 [==============================] - 0s 166ms/step - loss: 0.5368 - val_loss: 0.7469
Epoch 45/1000
1/1 [==============================] - 0s 144ms/step - loss: 0.6078 - val_loss: 0.7469
Epoch 46/1000
1/1 [==============================] - 0s 185ms/step - loss: 0.6051 - val_loss: 0.7440
Epoch 47/1000
1/1 [==============================] - 0s 170ms/step - loss: 0.6545 - val_loss: 0.7431
Epoch 48/1000
1/1 [==============================] - 0s 143ms/step - loss: 0.7159 - val_loss: 0.7427
Epoch 49/1000
1/1 [==============================] - 0s 185ms/step - loss: 0.6826 - val_loss: 0.7421
Epoch 50/1000
1/1 [==============================] - 0s 169ms/step - loss: 0.5714 - val_loss: 0.7435
Epoch 51/1000
1/1 [==============================] - 0s 140ms/step - loss: 0.6145 - val_loss: 0.7434
Epoch 52/1000
1/1 [==============================] - 0s 183ms/step - loss: 0.6355 - val_loss: 0.7417
Epoch 53/1000
1/1 [==============================] - 0s 166ms/step - loss: 0.7310 - val_loss: 0.7402
Epoch 54/1000
1/1 [==============================] - 0s 146ms/step - loss: 0.5652 - val_loss: 0.7376
Epoch 55/1000
1/1 [==============================] - 0s 197ms/step - loss: 0.5525 - val_loss: 0.7339
Epoch 56/1000
1/1 [==============================] - 0s 158ms/step - loss: 0.6439 - val_loss: 0.7297
Epoch 57/1000
1/1 [==============================] - 0s 136ms/step - loss: 0.5920 - val_loss: 0.7237
Epoch 58/1000
1/1 [==============================] - 0s 184ms/step - loss: 0.5235 - val_loss: 0.7138
Epoch 59/1000
1/1 [==============================] - 0s 183ms/step - loss: 0.5966 - val_loss: 0.7075
Epoch 60/1000
1/1 [==============================] - 0s 167ms/step - loss: 0.5251 - val_loss: 0.6991
Epoch 61/1000
1/1 [==============================] - 0s 151ms/step - loss: 0.5401 - val_loss: 0.6915
Epoch 62/1000
1/1 [==============================] - 0s 180ms/step - loss: 0.6159 - val_loss: 0.6837
Epoch 63/1000
1/1 [==============================] - 0s 163ms/step - loss: 0.6649 - val_loss: 0.6891
Epoch 64/1000
1/1 [==============================] - 0s 146ms/step - loss: 0.5873 - val_loss: 0.6972
Epoch 65/1000
1/1 [==============================] - 0s 184ms/step - loss: 0.5517 - val_loss: 0.7073
Epoch 66/1000
1/1 [==============================] - 0s 165ms/step - loss: 0.6629 - val_loss: 0.7156
Epoch 67/1000
1/1 [==============================] - 0s 152ms/step - loss: 0.6376 - val_loss: 0.7243
Epoch 68/1000
1/1 [==============================] - 0s 177ms/step - loss: 0.5223 - val_loss: 0.7286
Epoch 69/1000
1/1 [==============================] - 0s 164ms/step - loss: 0.6337 - val_loss: 0.7303
Epoch 70/1000
1/1 [==============================] - 0s 146ms/step - loss: 0.5974 - val_loss: 0.7289
Epoch 71/1000
1/1 [==============================] - 0s 182ms/step - loss: 0.5760 - val_loss: 0.7242
Epoch 72/1000
1/1 [==============================] - 0s 164ms/step - loss: 0.5786 - val_loss: 0.7192
Epoch 73/1000
1/1 [==============================] - 0s 148ms/step - loss: 0.5882 - val_loss: 0.7114
Epoch 74/1000
1/1 [==============================] - 0s 186ms/step - loss: 0.6510 - val_loss: 0.7032
Epoch 75/1000
1/1 [==============================] - 0s 164ms/step - loss: 0.5840 - val_loss: 0.6983
Epoch 76/1000
1/1 [==============================] - 0s 154ms/step - loss: 0.5720 - val_loss: 0.6994
Epoch 77/1000
1/1 [==============================] - 0s 194ms/step - loss: 0.4894 - val_loss: 0.7010
Epoch 78/1000
1/1 [==============================] - 0s 160ms/step - loss: 0.5650 - val_loss: 0.7009
Epoch 79/1000
1/1 [==============================] - 0s 138ms/step - loss: 0.5772 - val_loss: 0.6996
Epoch 80/1000
1/1 [==============================] - 0s 185ms/step - loss: 0.6157 - val_loss: 0.7007
Epoch 81/1000
1/1 [==============================] - 0s 170ms/step - loss: 0.6975 - val_loss: 0.7047
Epoch 82/1000
1/1 [==============================] - 0s 139ms/step - loss: 0.6749 - val_loss: 0.7072
# Applies the trained LSTM model onto the dataframe
anomalies_df_PJMG = model.apply_model(time_PJMG, TIME_STEPS, input_tuple = (lstm_model_pjmg, scaler_pjmg))
anomalies_df_PJMG.to_csv('PJMG_Anomaly_data.csv')
print(anomalies_df_PJMG)
time loss threshold anomaly
DATE_INSERTED
2018-07-09 09:00:00 -2.105732 3.482354 3.367853 True
2018-07-13 21:00:00 -1.826748 3.201840 3.367853 False
2018-07-16 12:00:00 -2.035986 3.341193 3.367853 False
2018-07-20 21:00:00 -1.826748 3.135249 3.367853 False
2018-07-23 09:00:00 -2.105732 3.369841 3.367853 True
2018-07-27 21:00:00 -1.826748 3.124290 3.367853 False
2018-07-30 09:00:00 -2.105732 3.354602 3.367853 False
2018-07-30 12:00:00 -2.035986 3.332167 3.367853 False
2018-08-03 21:00:00 -1.826748 3.138281 3.367853 False
2018-08-06 12:00:00 -2.035986 3.299375 3.367853 False
2018-08-06 15:00:00 -1.966240 3.260308 3.367853 False
2018-08-10 21:00:00 -1.826748 3.124187 3.367853 False
2018-08-13 09:00:00 -2.105732 3.362630 3.367853 False
2018-08-17 21:00:00 -1.826748 3.127410 3.367853 False
2018-08-20 09:00:00 -2.105732 3.365864 3.367853 False
2018-08-20 15:00:00 -1.966240 3.271059 3.367853 False
latest_start = model.anomaly_schedule(anomalies_df_PJMG, TIME_STEPS)
print("Suggested time: ", anomalies_df_PJMG.index[latest_start], " to ", anomalies_df_PJMG.index[-1])
Suggested time: 2018-07-20 21:00:00 to 2018-08-20 15:00:00
# for PJM_X.csv
# Get the aggregated dataframe
parse_PJMX = parse_tb(PJM_X_Data)
resampled_PJMX = resample_df(parse_PJMX, 'H')
time_PJMX = get_hour_df(drop_zero_df(resampled_PJMX))
# Amount of time units (days, hours etc.) to look back on
TIME_STEPS = 10
# Creates and saves the model and scaler transformation
model = LSTMAE()
# Note: this retrains the model, and will take substantial amount of time
lstm_model_pjmx, scaler_pjmx = model.create_model(time_PJMX, TIME_STEPS)
Epoch 1/1000
10/10 [==============================] - 5s 236ms/step - loss: 0.6444 - val_loss: 0.6653
Epoch 2/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.4128 - val_loss: 0.6982
Epoch 3/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.4137 - val_loss: 0.6245
Epoch 4/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.3504 - val_loss: 0.6286
Epoch 5/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.3546 - val_loss: 0.5868
Epoch 6/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.3201 - val_loss: 0.5926
Epoch 7/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.3129 - val_loss: 0.5744
Epoch 8/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.3043 - val_loss: 0.5620
Epoch 9/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2896 - val_loss: 0.5619
Epoch 10/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.3109 - val_loss: 0.5540
Epoch 11/1000
10/10 [==============================] - 2s 196ms/step - loss: 0.2922 - val_loss: 0.5534
Epoch 12/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2827 - val_loss: 0.5530
Epoch 13/1000
10/10 [==============================] - 2s 196ms/step - loss: 0.2769 - val_loss: 0.5517
Epoch 14/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.2845 - val_loss: 0.5513
Epoch 15/1000
10/10 [==============================] - 2s 193ms/step - loss: 0.2867 - val_loss: 0.5547
Epoch 16/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2784 - val_loss: 0.5613
Epoch 17/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.3158 - val_loss: 0.5512
Epoch 18/1000
10/10 [==============================] - 2s 202ms/step - loss: 0.3433 - val_loss: 0.6139
Epoch 19/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2924 - val_loss: 0.5809
Epoch 20/1000
10/10 [==============================] - 2s 193ms/step - loss: 0.3173 - val_loss: 0.5346
Epoch 21/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.3026 - val_loss: 0.5199
Epoch 22/1000
10/10 [==============================] - 2s 202ms/step - loss: 0.2826 - val_loss: 0.5247
Epoch 23/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2635 - val_loss: 0.5218
Epoch 24/1000
10/10 [==============================] - 2s 193ms/step - loss: 0.2798 - val_loss: 0.5179
Epoch 25/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2808 - val_loss: 0.5156
Epoch 26/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2681 - val_loss: 0.5121
Epoch 27/1000
10/10 [==============================] - 2s 200ms/step - loss: 0.2662 - val_loss: 0.5231
Epoch 28/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2562 - val_loss: 0.5188
Epoch 29/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2770 - val_loss: 0.5132
Epoch 30/1000
10/10 [==============================] - 2s 200ms/step - loss: 0.3149 - val_loss: 0.5083
Epoch 31/1000
10/10 [==============================] - 2s 193ms/step - loss: 0.2720 - val_loss: 0.5228
Epoch 32/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2819 - val_loss: 0.5350
Epoch 33/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2725 - val_loss: 0.5180
Epoch 34/1000
10/10 [==============================] - 2s 201ms/step - loss: 0.2593 - val_loss: 0.5188
Epoch 35/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2481 - val_loss: 0.5028
Epoch 36/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2457 - val_loss: 0.4993
Epoch 37/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2534 - val_loss: 0.4913
Epoch 38/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2873 - val_loss: 0.4987
Epoch 39/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2738 - val_loss: 0.5259
Epoch 40/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2603 - val_loss: 0.4947
Epoch 41/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2646 - val_loss: 0.5101
Epoch 42/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2381 - val_loss: 0.5006
Epoch 43/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.2294 - val_loss: 0.4957
Epoch 44/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2288 - val_loss: 0.4893
Epoch 45/1000
10/10 [==============================] - 2s 201ms/step - loss: 0.2399 - val_loss: 0.5040
Epoch 46/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2456 - val_loss: 0.4865
Epoch 47/1000
10/10 [==============================] - 2s 195ms/step - loss: 0.2581 - val_loss: 0.4780
Epoch 48/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.2526 - val_loss: 0.5106
Epoch 49/1000
10/10 [==============================] - 2s 195ms/step - loss: 0.2463 - val_loss: 0.4741
Epoch 50/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2610 - val_loss: 0.4926
Epoch 51/1000
10/10 [==============================] - 2s 195ms/step - loss: 0.2391 - val_loss: 0.4905
Epoch 52/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2299 - val_loss: 0.4852
Epoch 53/1000
10/10 [==============================] - 2s 195ms/step - loss: 0.2333 - val_loss: 0.4765
Epoch 54/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2343 - val_loss: 0.4895
Epoch 55/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2386 - val_loss: 0.4715
Epoch 56/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2326 - val_loss: 0.5007
Epoch 57/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2355 - val_loss: 0.4884
Epoch 58/1000
10/10 [==============================] - 2s 195ms/step - loss: 0.2471 - val_loss: 0.4821
Epoch 59/1000
10/10 [==============================] - 2s 202ms/step - loss: 0.2304 - val_loss: 0.4835
Epoch 60/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2380 - val_loss: 0.4956
Epoch 61/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2381 - val_loss: 0.4757
Epoch 62/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.2222 - val_loss: 0.4954
Epoch 63/1000
10/10 [==============================] - 2s 191ms/step - loss: 0.2315 - val_loss: 0.4966
Epoch 64/1000
10/10 [==============================] - 2s 193ms/step - loss: 0.2389 - val_loss: 0.5038
Epoch 65/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.2282 - val_loss: 0.4845
Epoch 66/1000
10/10 [==============================] - 2s 196ms/step - loss: 0.2383 - val_loss: 0.4914
Epoch 67/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2261 - val_loss: 0.4964
Epoch 68/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2191 - val_loss: 0.4678
Epoch 69/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.2352 - val_loss: 0.4660
Epoch 70/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2241 - val_loss: 0.4938
Epoch 71/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2297 - val_loss: 0.4882
Epoch 72/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2231 - val_loss: 0.4861
Epoch 73/1000
10/10 [==============================] - 2s 195ms/step - loss: 0.2304 - val_loss: 0.4786
Epoch 74/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2184 - val_loss: 0.4947
Epoch 75/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2342 - val_loss: 0.4513
Epoch 76/1000
10/10 [==============================] - 2s 201ms/step - loss: 0.2229 - val_loss: 0.4920
Epoch 77/1000
10/10 [==============================] - 2s 195ms/step - loss: 0.2203 - val_loss: 0.4677
Epoch 78/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2326 - val_loss: 0.4508
Epoch 79/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2128 - val_loss: 0.4712
Epoch 80/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2444 - val_loss: 0.4576
Epoch 81/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2171 - val_loss: 0.4426
Epoch 82/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2172 - val_loss: 0.4540
Epoch 83/1000
10/10 [==============================] - 2s 193ms/step - loss: 0.2158 - val_loss: 0.4486
Epoch 84/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.2165 - val_loss: 0.4396
Epoch 85/1000
10/10 [==============================] - 2s 195ms/step - loss: 0.2228 - val_loss: 0.4800
Epoch 86/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2256 - val_loss: 0.4340
Epoch 87/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2216 - val_loss: 0.4405
Epoch 88/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2136 - val_loss: 0.4732
Epoch 89/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2229 - val_loss: 0.4387
Epoch 90/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2184 - val_loss: 0.4491
Epoch 91/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2138 - val_loss: 0.4386
Epoch 92/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2336 - val_loss: 0.4608
Epoch 93/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2156 - val_loss: 0.4503
Epoch 94/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2017 - val_loss: 0.4651
Epoch 95/1000
10/10 [==============================] - 2s 201ms/step - loss: 0.2191 - val_loss: 0.4479
Epoch 96/1000
10/10 [==============================] - 2s 193ms/step - loss: 0.2283 - val_loss: 0.4411
Epoch 97/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2084 - val_loss: 0.4426
Epoch 98/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2161 - val_loss: 0.4445
Epoch 99/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.2030 - val_loss: 0.4503
Epoch 100/1000
10/10 [==============================] - 2s 193ms/step - loss: 0.2177 - val_loss: 0.4571
Epoch 101/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2133 - val_loss: 0.4298
Epoch 102/1000
10/10 [==============================] - 2s 195ms/step - loss: 0.2193 - val_loss: 0.4579
Epoch 103/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.2140 - val_loss: 0.4450
Epoch 104/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2015 - val_loss: 0.4469
Epoch 105/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2047 - val_loss: 0.4359
Epoch 106/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2049 - val_loss: 0.4455
Epoch 107/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2198 - val_loss: 0.4469
Epoch 108/1000
10/10 [==============================] - 2s 195ms/step - loss: 0.2015 - val_loss: 0.4324
Epoch 109/1000
10/10 [==============================] - 2s 200ms/step - loss: 0.2033 - val_loss: 0.4462
Epoch 110/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2101 - val_loss: 0.4339
Epoch 111/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2055 - val_loss: 0.4666
Epoch 112/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.2119 - val_loss: 0.4316
Epoch 113/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2089 - val_loss: 0.4389
Epoch 114/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.1912 - val_loss: 0.4425
Epoch 115/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.1978 - val_loss: 0.4628
Epoch 116/1000
10/10 [==============================] - 2s 199ms/step - loss: 0.1993 - val_loss: 0.4324
Epoch 117/1000
10/10 [==============================] - 2s 196ms/step - loss: 0.2000 - val_loss: 0.4392
Epoch 118/1000
10/10 [==============================] - 2s 197ms/step - loss: 0.2075 - val_loss: 0.4383
Epoch 119/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.2051 - val_loss: 0.4312
Epoch 120/1000
10/10 [==============================] - 2s 198ms/step - loss: 0.2003 - val_loss: 0.4694
Epoch 121/1000
10/10 [==============================] - 2s 194ms/step - loss: 0.1999 - val_loss: 0.4341
anomalies_df_PJMX = model.apply_model(time_PJMX, TIME_STEPS, input_tuple = (lstm_model_pjmx, scaler_pjmx))
anomalies_df_PJMX.to_csv('pjmx_anomaly.csv')
print(anomalies_df_PJMX)
time loss threshold anomaly
DATE_INSERTED
2011-09-01 13:00:00 -0.269337 0.250951 0.663609 False
2011-09-02 13:00:00 -0.269337 0.256640 0.663609 False
2011-09-03 13:00:00 -0.269337 0.254454 0.663609 False
2011-09-04 13:00:00 -0.269337 0.180912 0.663609 False
2011-09-04 14:00:00 -0.098226 0.009801 0.663609 False
... ... ... ... ...
2014-05-31 14:00:00 -0.098226 0.071514 0.663609 False
2014-05-31 15:00:00 0.072885 0.040782 0.663609 False
2014-05-31 16:00:00 0.243996 0.069217 0.663609 False
2014-05-31 17:00:00 0.415107 0.048447 0.663609 False
2015-05-16 20:00:00 0.928439 0.354662 0.663609 False
[778 rows x 4 columns]
latest_start = model.anomaly_schedule(anomalies_df_PJMX, TIME_STEPS)
print("Suggested time: ", anomalies_df_PJMX.index[latest_start], " to ", anomalies_df_PJMX.index[-1])
Suggested time: 2014-05-27 23:00:00 to 2015-05-16 20:00:00
# for MISO_data
# Get the aggregated dataframe
parse_MISO = parse_tb(MISO_data)
resampled_MISO = resample_df(parse_MISO, 'H')
time_MISO = get_hour_df(drop_zero_df(resampled_MISO))
# Amount of time units (days, hours etc.) to look back on
TIME_STEPS = 10
# Creates and saves the model and scaler transformation
model = LSTMAE()
# Note: this retrains the model, and will take substantial amount of time
lstm_model_MISO, scaler_MISO = model.create_model(time_MISO, TIME_STEPS)
Epoch 1/1000
3/3 [==============================] - 4s 401ms/step - loss: 0.7792 - val_loss: 0.9172
Epoch 2/1000
3/3 [==============================] - 1s 202ms/step - loss: 0.4465 - val_loss: 0.4323
Epoch 3/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.3563 - val_loss: 0.5333
Epoch 4/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.3651 - val_loss: 0.3781
Epoch 5/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.2768 - val_loss: 0.4703
Epoch 6/1000
3/3 [==============================] - 1s 215ms/step - loss: 0.2942 - val_loss: 0.3938
Epoch 7/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.2595 - val_loss: 0.3131
Epoch 8/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.2429 - val_loss: 0.3265
Epoch 9/1000
3/3 [==============================] - 1s 215ms/step - loss: 0.2296 - val_loss: 0.2508
Epoch 10/1000
3/3 [==============================] - 1s 215ms/step - loss: 0.2103 - val_loss: 0.3491
Epoch 11/1000
3/3 [==============================] - 1s 231ms/step - loss: 0.2135 - val_loss: 0.2838
Epoch 12/1000
3/3 [==============================] - 1s 201ms/step - loss: 0.1994 - val_loss: 0.2090
Epoch 13/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1981 - val_loss: 0.2070
Epoch 14/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1867 - val_loss: 0.2304
Epoch 15/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.1916 - val_loss: 0.2418
Epoch 16/1000
3/3 [==============================] - 1s 218ms/step - loss: 0.2025 - val_loss: 0.1943
Epoch 17/1000
3/3 [==============================] - 1s 216ms/step - loss: 0.1871 - val_loss: 0.1924
Epoch 18/1000
3/3 [==============================] - 1s 207ms/step - loss: 0.1761 - val_loss: 0.2603
Epoch 19/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.1967 - val_loss: 0.1936
Epoch 20/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.1882 - val_loss: 0.1774
Epoch 21/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1825 - val_loss: 0.2569
Epoch 22/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.1886 - val_loss: 0.3316
Epoch 23/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1705 - val_loss: 0.1942
Epoch 24/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1804 - val_loss: 0.1586
Epoch 25/1000
3/3 [==============================] - 1s 215ms/step - loss: 0.1817 - val_loss: 0.1648
Epoch 26/1000
3/3 [==============================] - 1s 230ms/step - loss: 0.1820 - val_loss: 0.1797
Epoch 27/1000
3/3 [==============================] - 1s 205ms/step - loss: 0.1783 - val_loss: 0.1490
Epoch 28/1000
3/3 [==============================] - 1s 208ms/step - loss: 0.1637 - val_loss: 0.1465
Epoch 29/1000
3/3 [==============================] - 1s 209ms/step - loss: 0.1623 - val_loss: 0.1437
Epoch 30/1000
3/3 [==============================] - 1s 216ms/step - loss: 0.1595 - val_loss: 0.1412
Epoch 31/1000
3/3 [==============================] - 1s 218ms/step - loss: 0.1545 - val_loss: 0.2076
Epoch 32/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1679 - val_loss: 0.1366
Epoch 33/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1628 - val_loss: 0.1367
Epoch 34/1000
3/3 [==============================] - 1s 220ms/step - loss: 0.1598 - val_loss: 0.2200
Epoch 35/1000
3/3 [==============================] - 1s 217ms/step - loss: 0.1634 - val_loss: 0.1643
Epoch 36/1000
3/3 [==============================] - 1s 236ms/step - loss: 0.1692 - val_loss: 0.1295
Epoch 37/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.1612 - val_loss: 0.1520
Epoch 38/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1572 - val_loss: 0.1368
Epoch 39/1000
3/3 [==============================] - 1s 210ms/step - loss: 0.1475 - val_loss: 0.1220
Epoch 40/1000
3/3 [==============================] - 1s 221ms/step - loss: 0.1495 - val_loss: 0.1463
Epoch 41/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1433 - val_loss: 0.1537
Epoch 42/1000
3/3 [==============================] - 1s 215ms/step - loss: 0.1444 - val_loss: 0.1186
Epoch 43/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1361 - val_loss: 0.1518
Epoch 44/1000
3/3 [==============================] - 1s 216ms/step - loss: 0.1480 - val_loss: 0.1272
Epoch 45/1000
3/3 [==============================] - 1s 200ms/step - loss: 0.1403 - val_loss: 0.1177
Epoch 46/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1461 - val_loss: 0.1412
Epoch 47/1000
3/3 [==============================] - 1s 215ms/step - loss: 0.1419 - val_loss: 0.1094
Epoch 48/1000
3/3 [==============================] - 1s 229ms/step - loss: 0.1359 - val_loss: 0.1047
Epoch 49/1000
3/3 [==============================] - 1s 205ms/step - loss: 0.1317 - val_loss: 0.1817
Epoch 50/1000
3/3 [==============================] - 1s 206ms/step - loss: 0.1313 - val_loss: 0.1001
Epoch 51/1000
3/3 [==============================] - 1s 202ms/step - loss: 0.1211 - val_loss: 0.1066
Epoch 52/1000
3/3 [==============================] - 1s 206ms/step - loss: 0.1318 - val_loss: 0.0930
Epoch 53/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1308 - val_loss: 0.1519
Epoch 54/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1163 - val_loss: 0.0948
Epoch 55/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1243 - val_loss: 0.1175
Epoch 56/1000
3/3 [==============================] - 1s 217ms/step - loss: 0.1378 - val_loss: 0.2231
Epoch 57/1000
3/3 [==============================] - 1s 215ms/step - loss: 0.1272 - val_loss: 0.0910
Epoch 58/1000
3/3 [==============================] - 1s 210ms/step - loss: 0.1299 - val_loss: 0.1630
Epoch 59/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.1249 - val_loss: 0.0969
Epoch 60/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1292 - val_loss: 0.2136
Epoch 61/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.1186 - val_loss: 0.0894
Epoch 62/1000
3/3 [==============================] - 1s 232ms/step - loss: 0.1237 - val_loss: 0.1686
Epoch 63/1000
3/3 [==============================] - 1s 227ms/step - loss: 0.1112 - val_loss: 0.0876
Epoch 64/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.1130 - val_loss: 0.1794
Epoch 65/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1148 - val_loss: 0.1013
Epoch 66/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.1022 - val_loss: 0.1157
Epoch 67/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.1203 - val_loss: 0.1018
Epoch 68/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1152 - val_loss: 0.1293
Epoch 69/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1048 - val_loss: 0.1115
Epoch 70/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.1026 - val_loss: 0.0978
Epoch 71/1000
3/3 [==============================] - 1s 208ms/step - loss: 0.0993 - val_loss: 0.0765
Epoch 72/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.1039 - val_loss: 0.0959
Epoch 73/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.0988 - val_loss: 0.0774
Epoch 74/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.1052 - val_loss: 0.0859
Epoch 75/1000
3/3 [==============================] - 1s 210ms/step - loss: 0.0954 - val_loss: 0.1007
Epoch 76/1000
3/3 [==============================] - 1s 210ms/step - loss: 0.0935 - val_loss: 0.0853
Epoch 77/1000
3/3 [==============================] - 1s 217ms/step - loss: 0.0927 - val_loss: 0.0757
Epoch 78/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.0905 - val_loss: 0.0774
Epoch 79/1000
3/3 [==============================] - 1s 231ms/step - loss: 0.0882 - val_loss: 0.0741
Epoch 80/1000
3/3 [==============================] - 1s 210ms/step - loss: 0.0876 - val_loss: 0.0758
Epoch 81/1000
3/3 [==============================] - 1s 226ms/step - loss: 0.0834 - val_loss: 0.0824
Epoch 82/1000
3/3 [==============================] - 1s 203ms/step - loss: 0.0860 - val_loss: 0.0726
Epoch 83/1000
3/3 [==============================] - 1s 206ms/step - loss: 0.0857 - val_loss: 0.0671
Epoch 84/1000
3/3 [==============================] - 1s 207ms/step - loss: 0.0882 - val_loss: 0.0676
Epoch 85/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0921 - val_loss: 0.0812
Epoch 86/1000
3/3 [==============================] - 1s 210ms/step - loss: 0.1043 - val_loss: 0.0813
Epoch 87/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.0966 - val_loss: 0.0789
Epoch 88/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0964 - val_loss: 0.0676
Epoch 89/1000
3/3 [==============================] - 1s 215ms/step - loss: 0.1010 - val_loss: 0.0698
Epoch 90/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0891 - val_loss: 0.0944
Epoch 91/1000
3/3 [==============================] - 1s 216ms/step - loss: 0.0916 - val_loss: 0.0933
Epoch 92/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.0872 - val_loss: 0.0826
Epoch 93/1000
3/3 [==============================] - 1s 219ms/step - loss: 0.0913 - val_loss: 0.0822
Epoch 94/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.0864 - val_loss: 0.0688
Epoch 95/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.0839 - val_loss: 0.1060
Epoch 96/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.0810 - val_loss: 0.0891
Epoch 97/1000
3/3 [==============================] - 1s 219ms/step - loss: 0.0787 - val_loss: 0.0716
Epoch 98/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.0856 - val_loss: 0.0836
Epoch 99/1000
3/3 [==============================] - 1s 211ms/step - loss: 0.0801 - val_loss: 0.0788
Epoch 100/1000
3/3 [==============================] - 1s 230ms/step - loss: 0.0762 - val_loss: 0.0644
Epoch 101/1000
3/3 [==============================] - 1s 234ms/step - loss: 0.0706 - val_loss: 0.0712
Epoch 102/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0761 - val_loss: 0.0678
Epoch 103/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.0809 - val_loss: 0.0753
Epoch 104/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.0825 - val_loss: 0.0752
Epoch 105/1000
3/3 [==============================] - 1s 216ms/step - loss: 0.0768 - val_loss: 0.0671
Epoch 106/1000
3/3 [==============================] - 1s 208ms/step - loss: 0.0760 - val_loss: 0.0691
Epoch 107/1000
3/3 [==============================] - 1s 214ms/step - loss: 0.0760 - val_loss: 0.0642
Epoch 108/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.0694 - val_loss: 0.0677
Epoch 109/1000
3/3 [==============================] - 1s 216ms/step - loss: 0.0750 - val_loss: 0.0641
Epoch 110/1000
3/3 [==============================] - 1s 210ms/step - loss: 0.0720 - val_loss: 0.0662
Epoch 111/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0704 - val_loss: 0.0694
Epoch 112/1000
3/3 [==============================] - 1s 218ms/step - loss: 0.0649 - val_loss: 0.0741
Epoch 113/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0673 - val_loss: 0.0655
Epoch 114/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0689 - val_loss: 0.0690
Epoch 115/1000
3/3 [==============================] - 1s 206ms/step - loss: 0.0706 - val_loss: 0.0655
Epoch 116/1000
3/3 [==============================] - 1s 216ms/step - loss: 0.0805 - val_loss: 0.0837
Epoch 117/1000
3/3 [==============================] - 1s 228ms/step - loss: 0.0767 - val_loss: 0.0911
Epoch 118/1000
3/3 [==============================] - 1s 202ms/step - loss: 0.0669 - val_loss: 0.0670
Epoch 119/1000
3/3 [==============================] - 1s 203ms/step - loss: 0.0649 - val_loss: 0.0653
Epoch 120/1000
3/3 [==============================] - 1s 201ms/step - loss: 0.0628 - val_loss: 0.0636
Epoch 121/1000
3/3 [==============================] - 1s 201ms/step - loss: 0.0618 - val_loss: 0.0625
Epoch 122/1000
3/3 [==============================] - 1s 204ms/step - loss: 0.0642 - val_loss: 0.0655
Epoch 123/1000
3/3 [==============================] - 1s 215ms/step - loss: 0.0676 - val_loss: 0.0619
Epoch 124/1000
3/3 [==============================] - 1s 216ms/step - loss: 0.0713 - val_loss: 0.0664
Epoch 125/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.0761 - val_loss: 0.0669
Epoch 126/1000
3/3 [==============================] - 1s 210ms/step - loss: 0.0731 - val_loss: 0.0712
Epoch 127/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0645 - val_loss: 0.0658
Epoch 128/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.0620 - val_loss: 0.0655
Epoch 129/1000
3/3 [==============================] - 1s 210ms/step - loss: 0.0613 - val_loss: 0.0815
Epoch 130/1000
3/3 [==============================] - 1s 219ms/step - loss: 0.0615 - val_loss: 0.0773
Epoch 131/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.0625 - val_loss: 0.0723
Epoch 132/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0630 - val_loss: 0.0666
Epoch 133/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0689 - val_loss: 0.0709
Epoch 134/1000
3/3 [==============================] - 1s 217ms/step - loss: 0.0688 - val_loss: 0.0659
Epoch 135/1000
3/3 [==============================] - 1s 220ms/step - loss: 0.0668 - val_loss: 0.0638
Epoch 136/1000
3/3 [==============================] - 1s 212ms/step - loss: 0.0605 - val_loss: 0.0620
Epoch 137/1000
3/3 [==============================] - 1s 213ms/step - loss: 0.0546 - val_loss: 0.0622
Epoch 138/1000
3/3 [==============================] - 1s 234ms/step - loss: 0.0710 - val_loss: 0.0772
Epoch 139/1000
3/3 [==============================] - 1s 229ms/step - loss: 0.0568 - val_loss: 0.0817
Epoch 140/1000
3/3 [==============================] - 1s 204ms/step - loss: 0.0583 - val_loss: 0.0684
Epoch 141/1000
3/3 [==============================] - 1s 207ms/step - loss: 0.0575 - val_loss: 0.0629
Epoch 142/1000
3/3 [==============================] - 1s 201ms/step - loss: 0.0581 - val_loss: 0.0722
Epoch 143/1000
3/3 [==============================] - 1s 207ms/step - loss: 0.0622 - val_loss: 0.0709
anomalies_df_MISO = model.apply_model(time_MISO, TIME_STEPS, input_tuple = (lstm_model_MISO, scaler_MISO))
anomalies_df_MISO.to_csv('MISO_anomaly.csv')
print(anomalies_df_MISO)
time loss threshold anomaly
DATE_INSERTED
2020-06-23 00:00:00 -0.757429 0.040973 0.038922 True
2020-06-24 00:00:00 -0.757429 0.011980 0.038922 False
2020-06-25 00:00:00 -0.757429 0.015203 0.038922 False
2020-06-26 00:00:00 -0.757429 0.029458 0.038922 False
2020-06-27 00:00:00 -0.757429 0.041172 0.038922 True
... ... ... ... ...
2021-02-06 23:00:00 1.386525 0.002059 0.038922 False
2021-02-07 23:00:00 1.386525 0.002059 0.038922 False
2021-02-08 23:00:00 1.386525 0.002059 0.038922 False
2021-02-09 23:00:00 1.386525 0.002059 0.038922 False
2021-02-10 23:00:00 1.386525 0.002059 0.038922 False
[229 rows x 4 columns]
latest_start = model.anomaly_schedule(anomalies_df_MISO, TIME_STEPS)
print("Suggested time: ", anomalies_df_MISO.index[latest_start], " to ", anomalies_df_MISO.index[-1])
Suggested time: 2020-11-18 19:00:00 to 2021-02-10 23:00:00
# for PJM_A_Data
# Get the aggregated dataframe
parse_PJMG = parse_tb(PJM_G_Data)
resampled_PJMG = resample_df(parse_PJMG, 'H')
time_PJMG = get_hour_df(drop_zero_df(resampled_PJMG))
# Amount of time units (days, hours etc.) to look back on
TIME_STEPS = 10
# Creates and saves the model and scaler transformation
model = LSTMAE()
# Note: this retrains the model, and will take substantial amount of time
lstm_model_pjmg, scaler_pjmg = model.create_model(time_PJMG, TIME_STEPS)
#print(anomalies_df_geo.index)
geo_tb = anomalies_df_geo[anomalies_df_geo.index>'2017-11-13 15:00:00']
geo_no_anomaly = geo_tb[geo_tb['anomaly']==False]
geo_no_anomaly = geo_no_anomaly.drop(['anomaly'], axis =1)
print(geo_no_anomaly)
time loss threshold
DATE_INSERTED
2017-11-14 15:00:00 -1.066047 0.016566 0.020077
2017-11-15 15:00:00 -1.066047 0.002362 0.020077
2017-11-16 15:00:00 -1.066047 0.003521 0.020077
2017-11-17 15:00:00 -1.066047 0.008061 0.020077
2017-11-18 15:00:00 -1.066047 0.013807 0.020077
2017-11-20 15:00:00 -1.066047 0.013807 0.020077
2017-11-21 15:00:00 -1.066047 0.013807 0.020077
2017-11-22 15:00:00 -1.066047 0.013807 0.020077
2017-11-23 15:00:00 -1.066047 0.013807 0.020077
2017-11-24 15:00:00 -1.066047 0.013807 0.020077
2017-11-25 15:00:00 -1.066047 0.013807 0.020077
2017-11-27 15:00:00 -1.066047 0.013807 0.020077
2017-11-28 15:00:00 -1.066047 0.013807 0.020077
2017-11-29 15:00:00 -1.066047 0.013807 0.020077
2017-11-30 15:00:00 -1.066047 0.013807 0.020077
2017-12-01 15:00:00 -1.066047 0.013807 0.020077
2017-12-02 15:00:00 -1.066047 0.013807 0.020077
2017-12-03 15:00:00 -1.066047 0.013807 0.020077
2017-12-04 15:00:00 -1.066047 0.013807 0.020077
2017-12-05 15:00:00 -1.066047 0.013807 0.020077
2017-12-06 15:00:00 -1.066047 0.013807 0.020077
2017-12-07 15:00:00 -1.066047 0.013807 0.020077
2017-12-08 15:00:00 -1.066047 0.013807 0.020077
2017-12-09 15:00:00 -1.066047 0.013807 0.020077
2017-12-12 15:00:00 -1.066047 0.013807 0.020077
2017-12-13 15:00:00 -1.066047 0.013807 0.020077
2017-12-14 15:00:00 -1.066047 0.013807 0.020077
2017-12-15 15:00:00 -1.066047 0.013807 0.020077
2017-12-16 15:00:00 -1.066047 0.013807 0.020077
2017-12-18 15:00:00 -1.066047 0.013807 0.020077
2017-12-19 15:00:00 -1.066047 0.013807 0.020077
2017-12-20 15:00:00 -1.066047 0.013807 0.020077
2017-12-21 15:00:00 -1.066047 0.013807 0.020077
2017-12-22 15:00:00 -1.066047 0.013807 0.020077
2017-12-23 15:00:00 -1.066047 0.013807 0.020077
2017-12-26 15:00:00 -1.066047 0.013807 0.020077
2017-12-27 15:00:00 -1.066047 0.013807 0.020077
2017-12-28 15:00:00 -1.066047 0.013807 0.020077
2017-12-29 15:00:00 -1.066047 0.013807 0.020077
2017-12-30 15:00:00 -1.066047 0.013807 0.020077
2018-01-01 15:00:00 -1.066047 0.013807 0.020077
pplot.plot_with_crons(geo_tb, 'H', 10, cron_g, 'GEOGRAIN_CASH_BIDS', show_cron= True, show_missed = False)
hours_df_g = find_cron_hours(geo_no_anomaly,0)
days_dict_g = get_days(geo_no_anomaly, hours_df_g)
crons_g = create_crons(days_dict_g)
all_crons_g = get_interval_crons(crons_g,10)
pplot.plot_interval_crons(geo_tb, all_crons_g)