max/model-training
1
0
Fork 0
mirror of https://git.gfz-potsdam.de/naaice/model-training.git synced 2025-12-13 12:18:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

investigate scaling error

Browse Source
This commit is contained in:
Hannes Signer 2025-02-21 16:57:55 +01:00
parent c0ca00271e
commit 8051eb3c3d
2 changed files with 3790 additions and 430 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4164
src/POET_Training.ipynb
View File

File diff suppressed because one or more lines are too long

56
src/preprocessing.py
View File

@ -110,10 +110,17 @@ def custom_loss(preprocess, column_dict, h1, h2, h3, scaler_type="minmax", loss_
if scaler_type == "minmax": if scaler_type == "minmax":
predicted_inverse = predicted * scale_y + min_y predicted_inverse = predicted * scale_y + min_y
results_inverse = results * scale_X + min_X results_inverse = results * scale_X + min_X
# inverse standard scaling
elif scaler_type == "standard": elif scaler_type == "standard":
predicted_inverse = predicted * scale_y + mean_y predicted_inverse = predicted * scale_y + mean_y
results_inverse = results * scale_X + mean_X results_inverse = results * scale_X + mean_X
# apply exp1m on the columns of predicted_inverse and results_inverse
predicted_inverse = tf.math.expm1(predicted_inverse)
results_inverse = tf.math.expm1(results_inverse)
print(predicted_inverse)
# mass balance # mass balance
dBa = tf.keras.backend.abs( dBa = tf.keras.backend.abs(
@ -124,11 +131,6 @@ def custom_loss(preprocess, column_dict, h1, h2, h3, scaler_type="minmax", loss_
(predicted_inverse[:, column_dict["Sr"]] + predicted_inverse[:, column_dict["Celestite"]]) - (predicted_inverse[:, column_dict["Sr"]] + predicted_inverse[:, column_dict["Celestite"]]) -
(results_inverse[:, column_dict["Sr"]] + results_inverse[:, column_dict["Celestite"]]) (results_inverse[:, column_dict["Sr"]] + results_inverse[:, column_dict["Celestite"]])
) )
# H/O ratio has to be 2
# h2o_ratio = tf.keras.backend.abs(
# (predicted_inverse[:, column_dict["H"]] / predicted_inverse[:, column_dict["O"]]) - 2
# )
# huber loss # huber loss
huber_loss = tf.keras.losses.Huber(delta)(results, predicted) huber_loss = tf.keras.losses.Huber(delta)(results, predicted)
@ -184,31 +186,8 @@ def mass_balance_metric(preprocess, column_dict, scaler_type="minmax"):
def huber_metric(preprocess, scaler_type="minmax", delta=1.0): def huber_metric(preprocess, scaler_type="minmax", delta=1.0):
if scaler_type == "minmax": def huber(results, predicted):
scale_X = tf.convert_to_tensor(preprocess.scaler_X.scale_, dtype=tf.float32)
min_X = tf.convert_to_tensor(preprocess.scaler_X.min_, dtype=tf.float32)
scale_y = tf.convert_to_tensor(preprocess.scaler_y.scale_, dtype=tf.float32)
min_y = tf.convert_to_tensor(preprocess.scaler_y.min_, dtype=tf.float32)
elif scaler_type == "standard":
scale_X = tf.convert_to_tensor(preprocess.scaler_X.scale_, dtype=tf.float32)
mean_X = tf.convert_to_tensor(preprocess.scaler_X.mean_, dtype=tf.float32)
scale_y = tf.convert_to_tensor(preprocess.scaler_y.scale_, dtype=tf.float32)
mean_y = tf.convert_to_tensor(preprocess.scaler_y.mean_, dtype=tf.float32)
def huber(results, predicted):
if scaler_type == "minmax":
predicted_inverse = predicted * scale_y + min_y
results_inverse = results * scale_X + min_X
elif scaler_type == "standard":
predicted_inverse = predicted * scale_y + mean_y
results_inverse = results * scale_X + mean_X
huber_loss = tf.keras.losses.Huber(delta)(results, predicted) huber_loss = tf.keras.losses.Huber(delta)(results, predicted)
return huber_loss return huber_loss
return huber return huber
@ -245,8 +224,8 @@ class preprocessing:
def funcTranform(self, X, y): def funcTranform(self, X, y):
for key in X.keys(): for key in X.keys():
if "Class" not in key: if "Class" not in key:
X[key] = X[key].apply(self.func_dict_in[key]) X[key] = X[key].apply(self.func_dict_in)
y[key] = y[key].apply(self.func_dict_in[key]) y[key] = y[key].apply(self.func_dict_in)
self.state["log"] = True self.state["log"] = True
return X, y return X, y
@ -255,8 +234,8 @@ class preprocessing:
for key in X.keys(): for key in X.keys():
if "Class" not in key: if "Class" not in key:
X[key] = X[key].apply(self.func_dict_out[key]) X[key] = X[key].apply(self.func_dict_out)
y[key] = y[key].apply(self.func_dict_out[key]) y[key] = y[key].apply(self.func_dict_out)
self.state["log"] = False self.state["log"] = False
return X, y return X, y
@ -363,7 +342,8 @@ class preprocessing:
def scale_inverse(self, X): def scale_inverse(self, X):
if("Class" in X.columns): if("Class" in X.columns):
X = pd.concat([self.scaler_X.inverse_transform(X.loc[:, X.columns != "Class"]), X.loc[:, "Class"]], axis=1) print("Class column found")
X = pd.concat([pd.DataFrame(self.scaler_X.inverse_transform(X.loc[:, X.columns != "Class"]), columns=X.columns[:-1]), X.loc[:, "Class"]], axis=1)
else: else:
X = self.scaler_X.inverse_transform(X) X = self.scaler_X.inverse_transform(X)
@ -374,6 +354,12 @@ class preprocessing:
return X_train, y_train, X_test, y_test return X_train, y_train, X_test, y_test
def class_selection(self, X, y, class_label):
X = X[X['Class'] == class_label]
y = y[y['Class'] == class_label]
return X, y