max/model-training
1
0
Fork 0
mirror of https://git.gfz-potsdam.de/naaice/model-training.git synced 2025-12-16 20:04:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: max:09a56875804609e730a50206ac5afb35af45d4f2
Branches Tags
max:main
max:mdl-dataset
max:box-cox
..
compare: max:f2c89e0b83b6891a5d0d8a99b75441185263a70d
Branches Tags
max:mdl-dataset
max:box-cox
max:main

No commits in common. "09a56875804609e730a50206ac5afb35af45d4f2" and "f2c89e0b83b6891a5d0d8a99b75441185263a70d" have entirely different histories.
09a5687580 ... f2c89e0b83

6 changed files with 1328 additions and 403 deletions
Show Stats Expand all files Collapse all files

BIN
results/adam_history.pkl (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
results/rmsprop_history.pkl (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
results/sgd_history.pkl (Stored with Git LFS) Normal file
View File

Binary file not shown.

1298
src/POET_Training.ipynb
View File

File diff suppressed because one or more lines are too long

410
src/preprocessing.py
View File

@ -160,29 +160,38 @@ def custom_loss(
preprocess.scaler_type, scaler_type)) preprocess.scaler_type, scaler_type))
if scaler_type == "minmax": if scaler_type == "minmax":
data_range = tf.convert_to_tensor( scale_X = tf.convert_to_tensor(
preprocess.scaler_output.data_range_, dtype=tf.float32 preprocess.scaler_X.data_range_, dtype=tf.float32
) )
min_values = tf.convert_to_tensor( min_X = tf.convert_to_tensor(
preprocess.scaler_output.data_min_, dtype=tf.float32 preprocess.scaler_X.data_min_, dtype=tf.float32
)
scale_y = tf.convert_to_tensor(
preprocess.scaler_y.data_range_, dtype=tf.float32
)
min_y = tf.convert_to_tensor(
preprocess.scaler_y.data_min_, dtype=tf.float32
) )
elif scaler_type == "standard": elif scaler_type == "standard":
scale_output = tf.convert_to_tensor( scale_X = tf.convert_to_tensor(
preprocess.scaler_output.scale_, dtype=tf.float32) preprocess.scaler_X.scale_, dtype=tf.float32)
mean_output = tf.convert_to_tensor( mean_X = tf.convert_to_tensor(
preprocess.scaler_output.mean_, dtype=tf.float32) preprocess.scaler_X.mean_, dtype=tf.float32)
scale_y = tf.convert_to_tensor(
preprocess.scaler_y.scale_, dtype=tf.float32)
mean_y = tf.convert_to_tensor(
preprocess.scaler_y.mean_, dtype=tf.float32)
def loss(results, predicted): def loss(results, predicted):
# inverse min/max scaling # inverse min/max scaling
if scaler_type == "minmax": if scaler_type == "minmax":
predicted_inverse = predicted * data_range + min_values predicted_inverse = predicted * scale_y + min_y
results_inverse = results * data_range + min_values results_inverse = results * scale_X + min_X
# inverse standard scaling # inverse standard scaling
elif scaler_type == "standard": elif scaler_type == "standard":
predicted_inverse = predicted * scale_output + mean_output predicted_inverse = predicted * scale_y + mean_y
results_inverse = results * scale_output + mean_output results_inverse = results * scale_X + mean_X
elif scaler_type == "none": elif scaler_type == "none":
predicted_inverse = predicted predicted_inverse = predicted
@ -195,8 +204,6 @@ def custom_loss(
# mass balance # mass balance
# in total no Barium and Strontium should be lost in one simulation step # in total no Barium and Strontium should be lost in one simulation step
# TODO: encapsulate the mass balance terms in a function
dBa = tf.keras.backend.abs( dBa = tf.keras.backend.abs(
( (
predicted_inverse[:, column_dict["Ba"]] predicted_inverse[:, column_dict["Ba"]]
@ -217,19 +224,6 @@ def custom_loss(
+ results_inverse[:, column_dict["Celestite"]] + results_inverse[:, column_dict["Celestite"]]
) )
) )
dS = tf.keras.backend.abs(
(
predicted_inverse[:, column_dict["S"]]
+ predicted_inverse[:, column_dict["Celestite"]]
+ predicted_inverse[:, column_dict["Barite"]]
)
- (
results_inverse[:, column_dict["S"]]
+ results_inverse[:, column_dict["Celestite"]]
+ results_inverse[:, column_dict["Barite"]]
)
)
# huber loss # huber loss
huber_loss = tf.keras.losses.Huber(delta)(results, predicted) huber_loss = tf.keras.losses.Huber(delta)(results, predicted)
@ -239,8 +233,6 @@ def custom_loss(
total_loss = huber_loss total_loss = huber_loss
elif loss_variant == "huber_mass_balance": elif loss_variant == "huber_mass_balance":
total_loss = h1 * huber_loss + h2 * dBa + h3 * dSr total_loss = h1 * huber_loss + h2 * dBa + h3 * dSr
elif "huber_mass_balance_extended":
total_loss = h1 * huber_loss + h2 * dBa + h3 * dSr + h3 * dS
else: else:
raise Exception( raise Exception(
"No valid loss variant found. Choose between 'huber' and 'huber_mass_balance'." "No valid loss variant found. Choose between 'huber' and 'huber_mass_balance'."
@ -251,7 +243,7 @@ def custom_loss(
return loss return loss
def mass_balance_metric(preprocess, column_dict, scaler_type="minmax", loss_variant="huber_mass_balance"): def mass_balance_metric(preprocess, column_dict, scaler_type="minmax"):
"""Auxilary function to calculate the mass balance during training. """Auxilary function to calculate the mass balance during training.
Args: Args:
@ -264,29 +256,36 @@ def mass_balance_metric(preprocess, column_dict, scaler_type="minmax", loss_vari
""" """
if scaler_type == "minmax": if scaler_type == "minmax":
data_range = tf.convert_to_tensor( scale_X = tf.convert_to_tensor(
preprocess.scaler_output.data_range_, dtype=tf.float32 preprocess.scaler_X.data_range_, dtype=tf.float32
) )
min_values = tf.convert_to_tensor( min_X = tf.convert_to_tensor(
preprocess.scaler_output.data_min_, dtype=tf.float32 preprocess.scaler_X.data_min_, dtype=tf.float32)
scale_y = tf.convert_to_tensor(
preprocess.scaler_y.data_range_, dtype=tf.float32
) )
min_y = tf.convert_to_tensor(
preprocess.scaler_y.data_min_, dtype=tf.float32)
elif scaler_type == "standard": elif scaler_type == "standard":
scale_output = tf.convert_to_tensor( scale_X = tf.convert_to_tensor(
preprocess.scaler_output.scale_, dtype=tf.float32) preprocess.scaler_X.scale_, dtype=tf.float32)
mean_output = tf.convert_to_tensor( mean_X = tf.convert_to_tensor(
preprocess.scaler_output.mean_, dtype=tf.float32) preprocess.scaler_X.mean_, dtype=tf.float32)
scale_y = tf.convert_to_tensor(
preprocess.scaler_y.scale_, dtype=tf.float32)
mean_y = tf.convert_to_tensor(
preprocess.scaler_y.mean_, dtype=tf.float32)
def mass_balance(results, predicted): def mass_balance(results, predicted):
# inverse min/max scaling # inverse min/max scaling
if scaler_type == "minmax": if scaler_type == "minmax":
predicted_inverse = predicted * data_range + min_values predicted_inverse = predicted * scale_y + min_y
results_inverse = results * data_range + min_values results_inverse = results * scale_X + min_X
# inverse standard scaling
elif scaler_type == "standard": elif scaler_type == "standard":
predicted_inverse = predicted * scale_output + mean_output predicted_inverse = predicted * scale_y + mean_y
results_inverse = results * scale_output + mean_output results_inverse = results * scale_X + mean_X
elif scaler_type == "none": elif scaler_type == "none":
predicted_inverse = predicted predicted_inverse = predicted
@ -307,7 +306,6 @@ def mass_balance_metric(preprocess, column_dict, scaler_type="minmax", loss_vari
+ results_inverse[:, column_dict["Barite"]] + results_inverse[:, column_dict["Barite"]]
) )
) )
dSr = tf.keras.backend.abs( dSr = tf.keras.backend.abs(
( (
predicted_inverse[:, column_dict["Sr"]] predicted_inverse[:, column_dict["Sr"]]
@ -318,74 +316,11 @@ def mass_balance_metric(preprocess, column_dict, scaler_type="minmax", loss_vari
+ results_inverse[:, column_dict["Celestite"]] + results_inverse[:, column_dict["Celestite"]]
) )
) )
return tf.reduce_mean(dBa + dSr)
dS = tf.keras.backend.abs(
(
predicted_inverse[:, column_dict["S"]]
+ predicted_inverse[:, column_dict["Celestite"]]
+ predicted_inverse[:, column_dict["Barite"]]
)
- (
results_inverse[:, column_dict["S"]]
+ results_inverse[:, column_dict["Celestite"]]
+ results_inverse[:, column_dict["Barite"]]
)
)
if loss_variant == "huber_mass_balance":
return tf.reduce_mean(dBa + dSr)
elif loss_variant == "huber_mass_balance_extended":
return tf.reduce_mean(dBa + dSr + dS)
return mass_balance return mass_balance
# def mass_balance_barium(predicted_inverse, results_inverse, column_dict):
# dBa = tf.keras.backend.abs(
# (
# predicted_inverse[:, column_dict["Ba"]]
# + predicted_inverse[:, column_dict["Barite"]]
# )
# - (
# results_inverse[:, column_dict["Ba"]]
# + results_inverse[:, column_dict["Barite"]]
# )
# )
# return dBa
# def mass_balance_strontium(predicted_inverse, results_inverse, column_dict):
# dSr = tf.keras.backend.abs(
# (
# predicted_inverse[:, column_dict["Sr"]]
# + predicted_inverse[:, column_dict["Celestite"]]
# )
# - (
# results_inverse[:, column_dict["Sr"]]
# + results_inverse[:, column_dict["Celestite"]]
# )
# )
# return dSr
# def mass_balance_sulfur(predicted_inverse, results_inverse, column_dict):
# dS = tf.keras.backend.abs(
# (
# predicted_inverse[:, column_dict["S"]]
# + predicted_inverse[:, column_dict["Celestite"]]
# + predicted_inverse[:, column_dict["Barite"]]
# )
# - (
# results_inverse[:, column_dict["S"]]
# + results_inverse[:, column_dict["Celestite"]]
# + results_inverse[:, column_dict["Barite"]]
# )
# )
# return dS
def huber_metric(delta=1.0): def huber_metric(delta=1.0):
"""Auxilary function to calculate the Huber loss during training. """Auxilary function to calculate the Huber loss during training.
@ -402,9 +337,8 @@ def huber_metric(delta=1.0):
return huber return huber
def mass_balance_evaluation(model, X, preprocess): def mass_balance_evaluation(model, X, y, preprocess):
"""Calculates the mass balance difference for each cell """Calculates the mass balance difference for each cell.
between the predicted values and the design dataset.
Args: Args:
model: trained model model: trained model
@ -419,12 +353,18 @@ def mass_balance_evaluation(model, X, preprocess):
columns = X.iloc[:, X.columns != "Class"].columns columns = X.iloc[:, X.columns != "Class"].columns
classes = X["Class"] classes = X["Class"]
classes.reset_index(drop=True, inplace=True) classes.reset_index(drop=True, inplace=True)
prediction = pd.DataFrame(model.predict(X[columns]), columns=preprocess.scaler_output.feature_names_in_) prediction = pd.DataFrame(model.predict(X[columns]), columns=y.columns)
# backtransform min/max or standard scaler # backtransform min/max or standard scaler
if preprocess.scaler_input is not None:
X = preprocess.scale_inverse(X)[0] if preprocess.scaler_X is not None:
prediction = preprocess.scale_inverse(prediction)[0] X = pd.DataFrame(
preprocess.scaler_X.inverse_transform(
X.iloc[:, X.columns != "Class"]),
columns=columns,
)
prediction = pd.DataFrame(
preprocess.scaler_y.inverse_transform(prediction), columns=columns
)
# apply backtransformation if log transformation was applied # apply backtransformation if log transformation was applied
if preprocess.func_dict_out is not None: if preprocess.func_dict_out is not None:
@ -438,12 +378,9 @@ def mass_balance_evaluation(model, X, preprocess):
(prediction["Sr"] + prediction["Celestite"]) - (prediction["Sr"] + prediction["Celestite"]) -
(X["Sr"] + X["Celestite"]) (X["Sr"] + X["Celestite"])
) )
dS = np.abs(
(prediction["S"] + prediction["Celestite"] + prediction["Barite"]) -
(X["S"] + X["Celestite"] + X["Barite"]))
mass_balance_result = pd.DataFrame( mass_balance_result = pd.DataFrame(
{"dBa": dBa, "dSr": dSr, "dS": dS, "mass_balance": dBa + dSr, "mass_balance_extended": dBa+dSr+dS, "Class": classes} {"dBa": dBa, "dSr": dSr, "mass_balance": dBa + dSr, "Class": classes}
) )
return mass_balance_result return mass_balance_result
@ -484,8 +421,8 @@ class preprocessing:
random_state (int, optional): Seed for reproducability. Defaults to 42. random_state (int, optional): Seed for reproducability. Defaults to 42.
""" """
self.random_state = random_state self.random_state = random_state
self.scaler_input = None self.scaler_X = None
self.scaler_output = None self.scaler_y = None
self.func_dict_in = func_dict_in if func_dict_in is not None else None self.func_dict_in = func_dict_in if func_dict_in is not None else None
self.func_dict_out = func_dict_out if func_dict_out is not None else None self.func_dict_out = func_dict_out if func_dict_out is not None else None
self.state = {"cluster": False, "log": False, self.state = {"cluster": False, "log": False,
@ -563,10 +500,8 @@ class preprocessing:
label = np.zeros(len(X)) label = np.zeros(len(X))
label[X[species] > threshold] = 1 label[X[species] > threshold] = 1
X = X.copy() X["Class"] = label
y = y.copy() y["Class"] = label
X.loc[:, "Class"] = label
y.loc[:, "Class"] = label
return X, y return X, y
@ -649,47 +584,52 @@ class preprocessing:
self.state["balance"] = True self.state["balance"] = True
return design_resampled, target_resampled return design_resampled, target_resampled
def scale_fit(self, X, y, type="standard"): def scale_fit(self, X, y, scaling, type="standard"):
self.scaler_type = type self.scaler_type = type
self.scaler_scope = scaling
"""Fit a scaler for data preprocessing. """Fit a scaler for data preprocessing.
Args: Args:
X: design dataset X: design dataset
y: target dataset y: target dataset
scaling: fit a scaler on all data in X and y. If X and y have different dimensions scaling: learn individual scaler for X and y when "individual" is selected or one global scaler on all data in X and y if "global" is selected (scaler_X and scaler_y are equal)
input and output scaler are trained for the specific columns.
type (str, optional): Using MinMax Scaling or Standarization. Defaults to "Standard". type (str, optional): Using MinMax Scaling or Standarization. Defaults to "Standard".
""" """
if type == "minmax": if type == "minmax":
self.scaler_input = MinMaxScaler() self.scaler_X = MinMaxScaler()
self.scaler_output = MinMaxScaler() self.scaler_y = MinMaxScaler()
elif type == "standard": elif type == "standard":
self.scaler_input = StandardScaler() self.scaler_X = StandardScaler()
self.scaler_output = StandardScaler() self.scaler_y = StandardScaler()
else: else:
raise Exception("No valid scaler type found") raise Exception("No valid scaler type found")
if scaling == "individual":
self.scaler_X.fit(X.iloc[:, X.columns != "Class"])
self.scaler_y.fit(y.iloc[:, y.columns != "Class"])
all_data = pd.concat([X, y],axis=0) elif scaling == "global":
self.scaler_X.fit(
if len(X.columns) == len(y.columns): pd.concat(
self.scaler_input.fit(all_data.loc[:, X.columns != "Class"]) [X.iloc[:, X.columns != "Class"],
self.scaler_output = self.scaler_input y.iloc[:, y.columns != "Class"]],
axis=0,
else: )
self.scaler_input.fit(all_data[X.columns[X.columns != "Class"]]) )
self.scaler_output.fit(all_data[y.columns[y.columns != "Class"]]) self.scaler_y = self.scaler_X
self.state["scale"] = True self.state["scale"] = True
return pd.concat(
[X.iloc[:, X.columns != "Class"],
y.iloc[:, y.columns != "Class"]],
axis=0,
)
def scale_transform(self, *args): def scale_transform(self, X_train, X_test, y_train, y_test):
"""Apply learned scaler on datasets. """Apply learned scaler on datasets.
Args: Args:
@ -701,31 +641,82 @@ class preprocessing:
Returns: Returns:
transformed dataframes transformed dataframes
""" """
if self.scaler_scope == "global":
if len(X_train.columns) > len(y_train.columns):
y_train_modified = X_train.copy()
y_test_modified = X_test.copy()
for i in y_train_modified.columns:
if i in y_train.columns:
y_train_modified[i] = y_train[i]
y_test_modified[i] = y_test[i]
else:
y_train_modified[i] = np.nan
y_test_modified[i] = np.nan
y_train = y_train_modified
y_test = y_test_modified
else:
X_train_modified = y_train.copy()
X_test_modified = y_test.copy()
for i in X_train_modified.columns:
if i in X_train.columns:
X_train_modified[i] = X_train[i]
X_test_modified[i] = X_test[i]
else:
X_train_modified[i] = np.nan
X_test_modified[i] = np.nan
X_train = X_train_modified
X_test = X_test_modified
X_train = pd.concat(
[
self.scaler_X.transform(
X_train.loc[:, X_train.columns != "Class"]),
X_train.loc[:, "Class"],
],
axis=1,
)
X_test = pd.concat(
[
self.scaler_X.transform(
X_test.loc[:, X_test.columns != "Class"]),
X_test.loc[:, "Class"],
],
axis=1,
)
y_train = pd.concat(
[
self.scaler_y.transform(
y_train.loc[:, y_train.columns != "Class"]),
y_train.loc[:, "Class"],
],
axis=1,
)
y_test = pd.concat(
[
self.scaler_y.transform(
y_test.loc[:, y_test.columns != "Class"]),
y_test.loc[:, "Class"],
],
axis=1,
)
results = [] X_train.dropna(axis=1, inplace=True)
for i in args: X_test.dropna(axis=1, inplace=True)
# check which scaler should be used depending on the columns y_train.dropna(axis=1, inplace=True)
if len(i.columns[i.columns != "Class"]) == len(self.scaler_input.feature_names_in_): y_test.dropna(axis=1, inplace=True)
scaler = self.scaler_input
else: return X_train, X_test, y_train, y_test
scaler = self.scaler_output
if "Class" in i.columns:
i = pd.concat(
[
scaler.transform(i.loc[:, i.columns != "Class"]),
i.loc[:, "Class"],
],
axis=1,
)
else:
i = scaler.transform(i)
results.append(i)
return results
def scale_inverse(self, *args): def scale_inverse(self, *args):
"""Backtransform the dataset """Backtransform the dataset
@ -734,28 +725,65 @@ class preprocessing:
Backtransformed data frames Backtransformed data frames
""" """
result = [] result = []
for i in args: if self.scaler_scope == "individual":
# check which scaler should be used depending on the columns for i in args:
if len(i.columns[i.columns != "Class"]) == len(self.scaler_input.feature_names_in_): if(len(i.columns) == len(self.scaler_X.feature_names_in_)):
scaler = self.scaler_input scaler = self.scaler_X
else: else:
scaler = self.scaler_output scaler = self.scaler_y
if "Class" in i.columns:
if "Class" in i.columns: inversed = pd.DataFrame(
inversed = pd.DataFrame( scaler.inverse_transform(
scaler.inverse_transform( i.loc[:, i.columns != "Class"]),
i.loc[:, i.columns != "Class"]), columns=i.columns[:-1],
columns=i.columns[:-1], )
) class_column = i.loc[:, "Class"].reset_index(drop=True)
class_column = i.loc[:, "Class"].reset_index(drop=True) i = pd.concat([inversed, class_column], axis=1)
i = pd.concat([inversed, class_column], axis=1) else:
else:
i = pd.DataFrame( i = pd.DataFrame(
scaler.inverse_transform(i), columns=i.columns) scaler.inverse_transform(i), columns=i.columns)
result.append(i) result.append(i)
elif self.scaler_scope == "global":
for i in args:
if (len(i.columns) == len(self.preprocess.scaler_X.feature_names_in_)):
if "Class" in i.columns:
inversed = pd.DataFrame(
self.scaler_X.inverse_transform(
i.loc[:, i.columns != "Class"]),
columns=i.columns[:-1],
)
class_column = i.loc[:, "Class"].reset_index(drop=True)
i = pd.concat([inversed, class_column], axis=1)
else:
i = pd.DataFrame(
self.scaler_X.inverse_transform(i), columns=i.columns)
result.append(i)
else:
df = pd.DataFrame()
for j in self.scaler_X.feature_names_in_:
if j in i.columns:
df[j] = i[j]
else:
df[j] = np.nan
if "Class" in i.columns:
inversed = pd.DataFrame(
self.scaler_X.inverse_transform(
df.loc[:, df.columns != "Class"]),
columns=df.columns[:-1],
)
else:
i = pd.DataFrame(
self.scaler_X.inverse_transform(df), columns=df.columns)
result.append(i)
return result return result

14
tests/test_scaler.py
View File

@ -1,14 +0,0 @@
# import unittest
# import os
# os.chdir("../src/")
# print(os.getcwd())
# from preprocessing import *
# class TestScaler(unittest.TestCase):
# def sample_test(self):
# self.assertEqual(1, 1)
# if __name__ == '__main__':
# unittest.main()