From db3ec236da0a2848e3adb09b1f7c13847be1dc06 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Mon, 3 Apr 2017 13:28:45 +0200 Subject: [PATCH 01/27] merge --- config.template.yaml | 2 ++ main.py | 2 ++ model/model.py | 46 ++++++++++++++++++++++++++++++++++++++++ model/util/csv_reader.py | 2 +- model/util/data.py | 5 +++++ 5 files changed, 56 insertions(+), 1 deletion(-) diff --git a/config.template.yaml b/config.template.yaml index 115f562..3ff473a 100644 --- a/config.template.yaml +++ b/config.template.yaml @@ -11,6 +11,7 @@ network: validation_data: 'validation_data_top_n_single.csv' training_data: 'training_data_top_n_single.csv' testing_data: 'testing_data_top_n_single.csv' + pre_train_data: 'pre-train_data_top_n.csv' # Embedding matrix configs: embedding_size: 150 # Make sure to match pretrained matrix dimensions trainable_matrix: true @@ -24,6 +25,7 @@ network: hidden_layers: 0 hidden_neurons: 300 use_concat_input: false + pre_train_subreddit: false # Regularisation configs: use_l2_loss: false l2_factor: 0.01 diff --git a/main.py b/main.py index dd048e3..d2c06d6 100644 --- a/main.py +++ b/main.py @@ -39,6 +39,8 @@ def main(): with tf.Session() as sess: builder = ModelBuilder(config_file, sess) network_model = builder.build() + if config_file["pre-train-subreddit"]: + network_model.pre_train() network_model.train() network_model.close_writers() tf.reset_default_graph() diff --git a/model/model.py b/model/model.py index 10fe2ea..9d5b0e4 100644 --- a/model/model.py +++ b/model/model.py @@ -231,6 +231,52 @@ def train(self): epoch_top=self.epoch_top, prec_valid=self.prec_valid, prec_train=self.prec_train, recall_valid=self.recall_valid, recall_train=self.recall_train) + def pre_train(self): + """ Pre-trains the model on the pre-training dataset """ + print("Starting pre-training...") + + old_epoch = 0 + + if self.use_pretrained: + self._session.run(self.embedding_init, feed_dict={ + self.embedding_placeholder: + self.data.embedding_matrix}) + + tmp_data, tmp_labels = self.data.get_training() + tmp_size = self.data.train_size + + self.data.train_data = self.data.pre_train_data + self.data.train_labels = self.data.pre_train_data + self.data.train_size = self.data.pre_train_size + + # Train for a specified amount of epochs + for i in self.data.for_n_train_epochs(self.training_epochs, + self.batch_size): + # Debug print out + epoch = self.data.completed_training_epochs + training_error = self.train_batch() + # validation_error = self.validate_batch() + + # Don't validate so often + if i % (self.data.train_size // self.batch_size // 10) == 0 and i: + done = self.data.percent_of_epoch + print( + "Validation error: {:f} | Training error: {:f} | Done: {:.0%}" + .format(0, training_error, done)) + + # Do a full evaluation once an epoch is complete + if epoch != old_epoch: + self._session.run(self.epoch.assign_add(1)) + print("Epoch complete...old ", old_epoch) + self.save_checkpoint() + # self.validate() + old_epoch = epoch + + self.data.train_data = tmp_data + self.data.train_labels = tmp_labels + self.data.train_size = tmp_size + self.data.completed_training_epochs = 0 + def train_batch(self): """ Trains for one batch and returns cross entropy error """ with tf.device("/cpu:0"): diff --git a/model/util/csv_reader.py b/model/util/csv_reader.py index c5b7526..9913c68 100644 --- a/model/util/csv_reader.py +++ b/model/util/csv_reader.py @@ -34,7 +34,7 @@ class Dataenum(Enum): TESTING = "testing_data" TRAINING = "training_data" VALIDATION = "validation_data" - + PRE_TRAINING = "pre_training_data" class CsvReader: def __init__(self, netcfg): diff --git a/model/util/data.py b/model/util/data.py index 4d9ac54..b50fb31 100644 --- a/model/util/data.py +++ b/model/util/data.py @@ -56,6 +56,11 @@ def __init__(self, networkconfig): def _read_data(self): """ Reads all the data from specified path """ + logging.debug("Reading pre-training data...") + + self.pre_train_data, self.pre_train_labels = self.reader.get_data(Dataenum.PRE_TRAINING) + self.pre_train_size = len(self.pre_train_data) + logging.debug("Reading training data...") self.train_data, self.train_subreddits, self.train_labels = \ From bc9154d45ebc502249b612d3ec0cc40b55044c46 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Tue, 4 Apr 2017 10:55:16 +0200 Subject: [PATCH 02/27] added method to add secondary output layer --- model/model_builder.py | 56 +++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/model/model_builder.py b/model/model_builder.py index a1b4f39..74ffa7a 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -52,6 +52,10 @@ def add_input_layer(self): tf.placeholder(tf.float64, [None, self._model.user_count], name="target") + self._model.sec_target = \ + tf.placeholder(tf.float64, + [None, self._model.data.subreddit_count], + name="sec_target") self._model.keep_prob = tf.placeholder(tf.float64, name="keep_prob") @@ -178,6 +182,49 @@ def add_output_layer(self): return self + def add_secondary_output(self): + """Adds a layer that can be used to train the network on data that is + labeled in a different way than the final data""" + # Output layer + # Feed the output of the previous layer to a sigmoid layer + sigmoid_weights = tf.Variable(tf.random_normal( + [self._model.latest_layer.get_shape()[1].value, + self._model.subreddit_count], + stddev=0.35, + dtype=tf.float64), + name="secondary_output_weights") + + sigmoid_bias = tf.Variable(tf.random_normal([self._model.subreddit_count], + stddev=0.35, + dtype=tf.float64), + name="secondary_output_biases") + + logits = tf.add(tf.matmul(self._model.latest_layer, sigmoid_weights), + sigmoid_bias) + # Training + + # Defne error function + error = tf.nn.sigmoid_cross_entropy_with_logits( + labels=self._model.sec_target, + logits=logits) + + if self._model.use_l2_loss: + cross_entropy = \ + tf.reduce_mean(tf.add( + tf.add(error, + tf.multiply(self._model.l2_factor, + tf.nn.l2_loss(sigmoid_weights))), + tf.add(tf.multiply(self._model.l2_factor, + tf.nn.l2_loss(sigmoid_bias)), + tf.multiply(self._model.l2_factor, + self._model.l2_term)))) + else: + cross_entropy = tf.reduce_mean(error) + + self._model.pre_train_op = tf.train.AdamOptimizer( + self._model.learning_rate).minimize(cross_entropy) + return self + def add_precision_operations(self): """Adds precision operation and tensorboard operations""" # Determine which prediction function to use. Casts a tensor to @@ -267,15 +314,6 @@ def add_precision_operations(self): def build(self): """Adds saver and init operation and returns the model""" - self.add_input_layer() - - # Add a number of hidden layers - for _ in range(self._model.hidden_layers): - self.add_layer(self._model.hidden_neurons) - - self.add_output_layer() - - self.add_precision_operations() self._model.init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) From 1afbf9f75a37c3f6616d82c22b0b4d535ad3f088 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Tue, 4 Apr 2017 10:58:53 +0200 Subject: [PATCH 03/27] implemented pre-training on subreddits --- main.py | 16 ++++++- model/model.py | 114 +++++++++++++++++++------------------------------ 2 files changed, 57 insertions(+), 73 deletions(-) diff --git a/main.py b/main.py index d2c06d6..68694b2 100644 --- a/main.py +++ b/main.py @@ -23,6 +23,7 @@ # ============================================================================== import argparse import tensorflow as tf +from definitions import * from model.util.networkconfig import yamlconfig as networkconfig from model.model_builder import ModelBuilder @@ -38,9 +39,20 @@ def main(): config_file = networkconfig[conf] with tf.Session() as sess: builder = ModelBuilder(config_file, sess) + builder.add_input_layer() + + # Add a number of hidden layers + for _ in range(config_file[HIDDEN_LAYERS]): + builder.add_layer(config_file[HIDDEN_NEURONS]) + + builder.add_output_layer()\ + .add_secondary_output()\ + .add_precision_operations() + network_model = builder.build() - if config_file["pre-train-subreddit"]: - network_model.pre_train() + if config_file[USE_PRETRAINED_NET]: + network_model.train(USE_PRETRAINED_NET) + network_model.data.completed_training_epochs = 0 network_model.train() network_model.close_writers() tf.reset_default_graph() diff --git a/model/model.py b/model/model.py index 9d5b0e4..4c11fa2 100644 --- a/model/model.py +++ b/model/model.py @@ -67,13 +67,17 @@ def __init__(self, config, session): self.use_constant_limit = config[USE_CONSTANT_LIMIT] self.constant_prediction_limit = config[CONSTANT_PREDICTION_LIMIT] self.use_concat_input = config[USE_CONCAT_INPUT] + self.use_pretrained_net = config[USE_PRETRAINED_NET] + self.subreddit_count = 0 # Will be set in build_graph self.input = None self.subreddit_input = None self.target = None + self.sec_target = None self.sigmoid = None self.train_op = None + self.pre_train_op = None self.error = None self.init_op = None self.saver = None @@ -105,6 +109,7 @@ def __init__(self, config, session): with tf.device("/cpu:0"): self.data = data.Data(config) + self.subreddit_count = self.data.subreddit_count if self.use_pretrained: self.vocabulary_size = len(self.data.embedding_matrix) @@ -184,7 +189,7 @@ def validate_batch(self): # TODO funktionen gör alldeles för mycket, # dela upp utskrift, beräkning och träning - def train(self): + def train(self, use_pretrained_net=False): """ Trains the model on the dataset """ print("Starting training...") @@ -200,29 +205,33 @@ def train(self): old_epoch = 0 - if self.epoch.eval(self._session) == 0: + if self.epoch.eval(self._session) == 0 and not use_pretrained_net: self.validate() # Train for a specified amount of epochs - for i in self.data.for_n_train_epochs(self.training_epochs, - self.batch_size): + for i in self.data.for_n_train_epochs(self.batch_size, self.training_epochs): # Debug print out epoch = self.data.completed_training_epochs - training_error = self.train_batch() - validation_error = self.validate_batch() - # Don't validate so often - if i % (self.data.train_size // self.batch_size // 10) == 0 and i: - done = self.data.percent_of_epoch - print("Validation error: {:f} | Training error: {:f} | Done: {:.0%}" - .format(validation_error, training_error, done)) + if not use_pretrained_net: + training_error = self.train_batch() + validation_error = self.validate_batch() + + # Don't validate so often + if i % (self.data.train_size // self.batch_size // 10) == 0 and i: + done = self.data.percent_of_epoch + print("Validation error: {:f} | Training error: {:f} | Done: {:.0%}" + .format(validation_error, training_error, done)) + else: + self.train_batch(True) # Do a full evaluation once an epoch is complete if epoch != old_epoch: self._session.run(self.epoch.assign_add(1)) print("Epoch complete...old ", old_epoch) self.save_checkpoint() - self.validate() + if not self.use_pretrained_net: + self.validate() old_epoch = epoch # Save model when done training @@ -231,67 +240,30 @@ def train(self): epoch_top=self.epoch_top, prec_valid=self.prec_valid, prec_train=self.prec_train, recall_valid=self.recall_valid, recall_train=self.recall_train) - def pre_train(self): - """ Pre-trains the model on the pre-training dataset """ - print("Starting pre-training...") - - old_epoch = 0 - - if self.use_pretrained: - self._session.run(self.embedding_init, feed_dict={ - self.embedding_placeholder: - self.data.embedding_matrix}) - - tmp_data, tmp_labels = self.data.get_training() - tmp_size = self.data.train_size - - self.data.train_data = self.data.pre_train_data - self.data.train_labels = self.data.pre_train_data - self.data.train_size = self.data.pre_train_size - - # Train for a specified amount of epochs - for i in self.data.for_n_train_epochs(self.training_epochs, - self.batch_size): - # Debug print out - epoch = self.data.completed_training_epochs - training_error = self.train_batch() - # validation_error = self.validate_batch() - - # Don't validate so often - if i % (self.data.train_size // self.batch_size // 10) == 0 and i: - done = self.data.percent_of_epoch - print( - "Validation error: {:f} | Training error: {:f} | Done: {:.0%}" - .format(0, training_error, done)) - - # Do a full evaluation once an epoch is complete - if epoch != old_epoch: - self._session.run(self.epoch.assign_add(1)) - print("Epoch complete...old ", old_epoch) - self.save_checkpoint() - # self.validate() - old_epoch = epoch - - self.data.train_data = tmp_data - self.data.train_labels = tmp_labels - self.data.train_size = tmp_size - self.data.completed_training_epochs = 0 - - def train_batch(self): + def train_batch(self, pre_train_net=False): """ Trains for one batch and returns cross entropy error """ with tf.device("/cpu:0"): - batch_input, batch_sub, batch_label = \ - self.data.next_train_batch() - - self._session.run(self.train_op, - {self.input: batch_input, - self.subreddit_input: batch_sub, - self.target: batch_label}) - - return self._session.run(self.error, - feed_dict={self.input: batch_input, - self.subreddit_input: batch_sub, - self.target: batch_label}) + if not pre_train_net: + batch_input, batch_sub, batch_label = \ + self.data.next_train_batch() + else: + batch_input, batch_label = \ + self.data.next_pre_train_batch() + + if pre_train_net: + self._session.run(self.pre_train_op, + {self.input: batch_input, + self.sec_target: batch_label}) + else: + self._session.run(self.train_op, + {self.input: batch_input, + self.subreddit_input: batch_sub, + self.target: batch_label}) + + return self._session.run(self.error, + feed_dict={self.input: batch_input, + self.subreddit_input: batch_sub, + self.target: batch_label}) def close_writers(self): """ Close tensorboard writers """ self.train_writer.close() From 95b779e619e7e6ce58c0882595517f8a263b4edc Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Tue, 4 Apr 2017 10:59:30 +0200 Subject: [PATCH 04/27] create methods for handling pre-training data --- definitions.py | 1 + model/util/data.py | 45 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/definitions.py b/definitions.py index fd70b22..b44179e 100644 --- a/definitions.py +++ b/definitions.py @@ -55,6 +55,7 @@ TRAINABLE_MATRIX = 'trainable_matrix' PRE_TRAINED_MATRIX = 'pre_trained_matrix' USE_PRETRAINED = 'use_pretrained' +USE_PRETRAINED_NET = 'pre_train_subreddit' VALIDATION_DATA = 'validation_data' TRAINING_DATA = 'training_data' TESTING_DATA = 'testing_data' diff --git a/model/util/data.py b/model/util/data.py index b50fb31..7300967 100644 --- a/model/util/data.py +++ b/model/util/data.py @@ -35,8 +35,11 @@ def __init__(self, networkconfig): self._current_train_index = 0 self._current_valid_index = 0 self._current_test_index = 0 + self._current_pre_train_index = 0 self.completed_training_epochs = 0 + self.completed_pre_training_epochs = 0 self.percent_of_epoch = 0.0 + self.subreddit_count = 0 self.title_length = networkconfig['max_title_length'] self.batch_size = self.netcfg['batch_size'] self.reader = CsvReader(networkconfig) @@ -58,8 +61,8 @@ def _read_data(self): """ Reads all the data from specified path """ logging.debug("Reading pre-training data...") - self.pre_train_data, self.pre_train_labels = self.reader.get_data(Dataenum.PRE_TRAINING) - self.pre_train_size = len(self.pre_train_data) + # self.pre_train_data, self.pre_train_labels = self.reader.get_data(Dataenum.TRAINING, label_column=1) + # self.pre_train_size = len(self.pre_train_data) logging.debug("Reading training data...") @@ -97,6 +100,7 @@ def _build_dict(self): subreddits = " ".join(self.train_subreddits).split() self.subreddit_dict = helper.build_subreddit_dict(subreddits) + self.subreddit_count = len(self.subreddit_dict) def next_train_batch(self, batch_size=None): """ Get the next batch of training data """ @@ -136,6 +140,39 @@ def next_train_batch(self, batch_size=None): self.percent_of_epoch = self._current_train_index / self.train_size return batch_x, batch_sub, batch_y + def next_pre_train_batch(self, batch_size=None): + """ Get the next batch of training data """ + batch_size = batch_size or self.batch_size + batch_x = [] + batch_y = [] + for _ in range(0, batch_size): + sentence = self.train_data[self._current_train_index] + label = self.train_labels[self._current_train_index] + self._current_pre_train_index += 1 + # Support multiple epochs + if self._current_pre_train_index >= self.train_size: + self._current_pre_train_index = 0 + self.completed_pre_training_epochs += 1 + self.percent_of_epoch = 0.0 + # TODO Detta ska inte ligga i funktionen som generar ny data + + # Turn sentences and labels into vector representations + sentence_vec, present, absent = \ + helper.get_indicies(sentence, + self.word_dict, + self.max_title_length) + self.train_present += present + self.train_absent += absent + + label_vec = helper.label_vector(label, + self.subreddit_dict, + self.subreddit_count) + batch_x.append(sentence_vec) + batch_y.append(label_vec) + + self.percent_of_epoch = self._current_train_index / self.train_size + return batch_x, batch_y + def get_validation(self): """ Get the whole validation set in a vectorized form """ old_ind = self._current_valid_index @@ -220,10 +257,10 @@ def next_test_batch(self, batch_size=None): batch_y.append(label_vec) return batch_x, batch_sub, batch_y - def for_n_train_epochs(self, num_epochs=1, batch_size=25): + def for_n_train_epochs(self, batch_size=25, num_epochs=1): # TODO Ta bort parameterar """ Calculates how many training iterations to do for num_epochs - number of epochs with a batch size of batch_size """ + number of epochs with a batch size of batch_size""" return range((self.train_size * num_epochs) // batch_size) def get_training(self): From 4afbf982ef56d345b55677662fcae6ea421d9ccc Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Tue, 4 Apr 2017 14:10:06 +0200 Subject: [PATCH 05/27] non-working commit to allow for debugg help. --- main.py | 1 - model/model.py | 11 ++++------- model/model_builder.py | 6 ++++++ model/util/data.py | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index 68694b2..78e9255 100644 --- a/main.py +++ b/main.py @@ -52,7 +52,6 @@ def main(): network_model = builder.build() if config_file[USE_PRETRAINED_NET]: network_model.train(USE_PRETRAINED_NET) - network_model.data.completed_training_epochs = 0 network_model.train() network_model.close_writers() tf.reset_default_graph() diff --git a/model/model.py b/model/model.py index 4c11fa2..ce61b13 100644 --- a/model/model.py +++ b/model/model.py @@ -193,15 +193,12 @@ def train(self, use_pretrained_net=False): """ Trains the model on the dataset """ print("Starting training...") - if self.use_pretrained: + if self.use_pretrained and \ + (self.use_pretrained_net and use_pretrained_net) or \ + (not self.use_pretrained_net and not use_pretrained_net): self._session.run(self.embedding_init, feed_dict={self.embedding_placeholder: self.data.embedding_matrix}) - self.train_writer = \ - tf.summary.FileWriter(self.logging_dir + '/' + TENSOR_DIR_TRAIN, - self._session.graph) - self.valid_writer = \ - tf.summary.FileWriter(self.logging_dir + '/' + TENSOR_DIR_VALID) old_epoch = 0 @@ -209,7 +206,7 @@ def train(self, use_pretrained_net=False): self.validate() # Train for a specified amount of epochs - for i in self.data.for_n_train_epochs(self.batch_size, self.training_epochs): + for i in self.data.for_n_train_epochs(self.training_epochs, self.batch_size): # Debug print out epoch = self.data.completed_training_epochs diff --git a/model/model_builder.py b/model/model_builder.py index 74ffa7a..39d27cb 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -24,6 +24,7 @@ import tensorflow as tf from model.model import Model +from definitions import * class ModelBuilder(object): """A class following the builder pattern to create a model""" @@ -315,6 +316,11 @@ def add_precision_operations(self): def build(self): """Adds saver and init operation and returns the model""" + self._model.train_writer = \ + tf.summary.FileWriter(self._model.logging_dir + '/' + TENSOR_DIR_TRAIN, + self._model._session.graph) + self._model.valid_writer = \ + tf.summary.FileWriter(self._model.logging_dir + '/' + TENSOR_DIR_VALID) self._model.init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) self._model.saver = tf.train.Saver() diff --git a/model/util/data.py b/model/util/data.py index 7300967..3da2866 100644 --- a/model/util/data.py +++ b/model/util/data.py @@ -257,7 +257,7 @@ def next_test_batch(self, batch_size=None): batch_y.append(label_vec) return batch_x, batch_sub, batch_y - def for_n_train_epochs(self, batch_size=25, num_epochs=1): + def for_n_train_epochs(self, num_epochs=1, batch_size=25): # TODO Ta bort parameterar """ Calculates how many training iterations to do for num_epochs number of epochs with a batch size of batch_size""" From 193811de992c561a8c88e09d62c1777ff6a23ec6 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 11:09:52 +0200 Subject: [PATCH 06/27] changed which variable was used to check if pre-training --- model/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/model.py b/model/model.py index ce61b13..998099e 100644 --- a/model/model.py +++ b/model/model.py @@ -227,7 +227,7 @@ def train(self, use_pretrained_net=False): self._session.run(self.epoch.assign_add(1)) print("Epoch complete...old ", old_epoch) self.save_checkpoint() - if not self.use_pretrained_net: + if not use_pretrained_net: self.validate() old_epoch = epoch From e5be63835298f1640f5f7bd7ddd2bf126a84263e Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 11:12:18 +0200 Subject: [PATCH 07/27] only add sec_output when pre-train and add an extra layer after --- main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 78e9255..2b63eaa 100644 --- a/main.py +++ b/main.py @@ -45,8 +45,11 @@ def main(): for _ in range(config_file[HIDDEN_LAYERS]): builder.add_layer(config_file[HIDDEN_NEURONS]) + if builder._model.use_pretrained_net: + builder.add_secondary_output() + builder.add_layer(config_file[HIDDEN_NEURONS]) + builder.add_output_layer()\ - .add_secondary_output()\ .add_precision_operations() network_model = builder.build() From d94d187f9dff5da850fdf53a3a767edce6971d84 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 17:38:52 +0200 Subject: [PATCH 08/27] changed config to not include unnecissary parameters --- config.template.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config.template.yaml b/config.template.yaml index 3ff473a..4b68e69 100644 --- a/config.template.yaml +++ b/config.template.yaml @@ -11,7 +11,6 @@ network: validation_data: 'validation_data_top_n_single.csv' training_data: 'training_data_top_n_single.csv' testing_data: 'testing_data_top_n_single.csv' - pre_train_data: 'pre-train_data_top_n.csv' # Embedding matrix configs: embedding_size: 150 # Make sure to match pretrained matrix dimensions trainable_matrix: true From fa9859cdee00e07cc730718ef0f854cad8ce515a Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 17:46:21 +0200 Subject: [PATCH 09/27] cleaned main.py by adding it to builder --- main.py | 14 +------------- model/model_builder.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/main.py b/main.py index 2b63eaa..6780e31 100644 --- a/main.py +++ b/main.py @@ -39,20 +39,8 @@ def main(): config_file = networkconfig[conf] with tf.Session() as sess: builder = ModelBuilder(config_file, sess) - builder.add_input_layer() - # Add a number of hidden layers - for _ in range(config_file[HIDDEN_LAYERS]): - builder.add_layer(config_file[HIDDEN_NEURONS]) - - if builder._model.use_pretrained_net: - builder.add_secondary_output() - builder.add_layer(config_file[HIDDEN_NEURONS]) - - builder.add_output_layer()\ - .add_precision_operations() - - network_model = builder.build() + network_model = builder.add_as_conf().build() if config_file[USE_PRETRAINED_NET]: network_model.train(USE_PRETRAINED_NET) network_model.train() diff --git a/model/model_builder.py b/model/model_builder.py index 39d27cb..14b243b 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -326,3 +326,17 @@ def build(self): self._model.saver = tf.train.Saver() self._model.load_checkpoint() return self._model + + def add_as_conf(self): + self.add_input_layer() + + # Add a number of hidden layers + for _ in range(self._model.hidden_layers): + self.add_layer(self._model.hidden_neurons) + + if self._model.use_pretrained_net: + self.add_secondary_output() + + self.add_output_layer() \ + .add_precision_operations() + return self From 1de1133b19762257809e361d5066135f50e2166f Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 17:50:16 +0200 Subject: [PATCH 10/27] sec_output uses softmax since a title only have one subreddit --- model/model_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/model_builder.py b/model/model_builder.py index 14b243b..c210b8a 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -205,7 +205,7 @@ def add_secondary_output(self): # Training # Defne error function - error = tf.nn.sigmoid_cross_entropy_with_logits( + error = tf.nn.softmax_cross_entropy_with_logits( labels=self._model.sec_target, logits=logits) From 5c2db6cfb133b7639c366b0087c8fe47a01e53dd Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 18:15:10 +0200 Subject: [PATCH 11/27] refactored add output method to one method --- model/model_builder.py | 77 ++++++++++++------------------------------ 1 file changed, 22 insertions(+), 55 deletions(-) diff --git a/model/model_builder.py b/model/model_builder.py index c210b8a..b16fc2d 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -138,77 +138,38 @@ def add_layer(self, number_of_neurons): return self - def add_output_layer(self): + def add_output_layer(self, output_size, secondary_output=False): """Adds an output layer, including error and optimisation functions. After this method no new layers should be added.""" # Output layer # Feed the output of the previous layer to a sigmoid layer sigmoid_weights = tf.Variable(tf.random_normal( - [self._model.latest_layer.get_shape()[1].value, self._model.user_count], + [self._model.latest_layer.get_shape()[1].value, output_size], stddev=0.35, dtype=tf.float64), name="output_weights") - sigmoid_bias = tf.Variable(tf.random_normal([self._model.user_count], + sigmoid_bias = tf.Variable(tf.random_normal([output_size], stddev=0.35, dtype=tf.float64), name="output_biases") logits = tf.add(tf.matmul(self._model.latest_layer, sigmoid_weights), sigmoid_bias) - self._model.sigmoid = tf.nn.sigmoid(logits) - # Training - - # Defne error function - error = tf.nn.sigmoid_cross_entropy_with_logits(labels=self._model.target, - logits=logits) - - if self._model.use_l2_loss: - cross_entropy = \ - tf.reduce_mean(tf.add( - tf.add(error, - tf.multiply(self._model.l2_factor, - tf.nn.l2_loss(sigmoid_weights))), - tf.add(tf.multiply(self._model.l2_factor, - tf.nn.l2_loss(sigmoid_bias)), - tf.multiply(self._model.l2_factor, - self._model.l2_term)))) + if secondary_output: + error = tf.nn.softmax_cross_entropy_with_logits( + labels=self._model.sec_target, + logits=logits) else: - cross_entropy = tf.reduce_mean(error) - - self._model.error = cross_entropy - self._model.train_op = tf.train.AdamOptimizer( - self._model.learning_rate).minimize(cross_entropy) + self._model.sigmoid = tf.nn.sigmoid(logits) + # Defne error function + error = tf.nn.sigmoid_cross_entropy_with_logits( + labels=self._model.target, + logits=logits) - return self - - def add_secondary_output(self): - """Adds a layer that can be used to train the network on data that is - labeled in a different way than the final data""" - # Output layer - # Feed the output of the previous layer to a sigmoid layer - sigmoid_weights = tf.Variable(tf.random_normal( - [self._model.latest_layer.get_shape()[1].value, - self._model.subreddit_count], - stddev=0.35, - dtype=tf.float64), - name="secondary_output_weights") - - sigmoid_bias = tf.Variable(tf.random_normal([self._model.subreddit_count], - stddev=0.35, - dtype=tf.float64), - name="secondary_output_biases") - - logits = tf.add(tf.matmul(self._model.latest_layer, sigmoid_weights), - sigmoid_bias) # Training - # Defne error function - error = tf.nn.softmax_cross_entropy_with_logits( - labels=self._model.sec_target, - logits=logits) - if self._model.use_l2_loss: cross_entropy = \ tf.reduce_mean(tf.add( @@ -222,8 +183,14 @@ def add_secondary_output(self): else: cross_entropy = tf.reduce_mean(error) - self._model.pre_train_op = tf.train.AdamOptimizer( - self._model.learning_rate).minimize(cross_entropy) + if secondary_output: + self._model.pre_train_op = tf.train.AdamOptimizer( + self._model.learning_rate).minimize(cross_entropy) + else: + self._model.error = cross_entropy + self._model.train_op = tf.train.AdamOptimizer( + self._model.learning_rate).minimize(cross_entropy) + return self def add_precision_operations(self): @@ -335,8 +302,8 @@ def add_as_conf(self): self.add_layer(self._model.hidden_neurons) if self._model.use_pretrained_net: - self.add_secondary_output() + self.add_output_layer(self._model.subreddit_count, secondary_output=True) - self.add_output_layer() \ + self.add_output_layer(self._model.user_count) \ .add_precision_operations() return self From 2374ed40c6ea268f213231f0cacc3ad202bd6cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20H=C3=A5kansson?= Date: Wed, 5 Apr 2017 11:40:56 +0200 Subject: [PATCH 12/27] Adds the option to choose between GRU and LSTM units --- config.template.yaml | 3 ++- definitions.py | 3 ++- model/model.py | 3 ++- model/model_builder.py | 10 ++++++++-- model/util/writer.py | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/config.template.yaml b/config.template.yaml index 115f562..1199a43 100644 --- a/config.template.yaml +++ b/config.template.yaml @@ -20,7 +20,8 @@ network: learning_rate: 0.5 training_epochs: 5 batch_size: 25 - lstm_neurons: 200 + rnn_neurons: 200 + rnn_unit: 'lstm' # Can be 'gru' or 'lstm', default: 'lstm' hidden_layers: 0 hidden_neurons: 300 use_concat_input: false diff --git a/definitions.py b/definitions.py index fd70b22..efdcf9e 100644 --- a/definitions.py +++ b/definitions.py @@ -40,7 +40,8 @@ LEARN_RATE = 'learning_rate' EMBEDD_SIZE = 'embedding_size' MAX_TITLE_LENGTH = 'max_title_length' -LSTM_NEURONS = 'lstm_neurons' +RNN_NEURONS = 'rnn_neurons' +RNN_UNIT = "rnn_unit" HIDDEN_NEURONS = 'hidden_neurons' HIDDEN_LAYERS = 'hidden_layers' USE_CONCAT_INPUT = 'use_concat_input' diff --git a/model/model.py b/model/model.py index 10fe2ea..c028882 100644 --- a/model/model.py +++ b/model/model.py @@ -53,7 +53,8 @@ def __init__(self, config, session): self.learning_rate = config[LEARN_RATE] self.embedding_size = config[EMBEDD_SIZE] self.max_title_length = config[MAX_TITLE_LENGTH] - self.lstm_neurons = config[LSTM_NEURONS] + self.rnn_neurons = config[RNN_NEURONS] + self.rnn_unit = config[RNN_UNIT] self.batch_size = config[BATCH_SIZE] self.training_epochs = config[TRAINING_EPOCHS] self.use_l2_loss = config[USE_L2_LOSS] diff --git a/model/model_builder.py b/model/model_builder.py index a1b4f39..02a5e5b 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -55,7 +55,13 @@ def add_input_layer(self): self._model.keep_prob = tf.placeholder(tf.float64, name="keep_prob") - lstm_layer = tf.contrib.rnn.LSTMCell(self._model.lstm_neurons, state_is_tuple=True) + if self._model.rnn_unit == 'lstm': + rnn_layer = tf.contrib.rnn.LSTMCell(self._model.rnn_neurons) + elif self._model.rnn_unit == 'gru': + rnn_layer = tf.contrib.rnn.GRUCell(self._model.rnn_neurons) + else: + print("Incorrect RNN unit, defaulting to LSTM") + rnn_layer = tf.contrib.rnn.LSTMCell(self._model.rnn_neurons) # Embedding matrix for the words embedding_matrix = tf.Variable( @@ -76,7 +82,7 @@ def add_input_layer(self): embedded_input = tf.nn.embedding_lookup(embedding_matrix, self._model.input) # Run the LSTM layer with the embedded input - outputs, _ = tf.nn.dynamic_rnn(lstm_layer, embedded_input, + outputs, _ = tf.nn.dynamic_rnn(rnn_layer, embedded_input, dtype=tf.float64) outputs = tf.transpose(outputs, [1, 0, 2]) diff --git a/model/util/writer.py b/model/util/writer.py index 09b8152..9b452ae 100644 --- a/model/util/writer.py +++ b/model/util/writer.py @@ -33,7 +33,7 @@ def log_samefile(config, f1_score_valid, f1_score_train, epoch_top, prec_valid, if not os.path.exists(LOGS_DIR): raise FileNotFoundError('Can not write because no directory is created') - config_headers = [NET_TYPE, NET_NAME, VOC_SIZE, USER_COUNT, LEARN_RATE, EMBEDD_SIZE, MAX_TITLE_LENGTH, LSTM_NEURONS, + config_headers = [NET_TYPE, NET_NAME, VOC_SIZE, USER_COUNT, LEARN_RATE, EMBEDD_SIZE, MAX_TITLE_LENGTH, RNN_NEURONS, RNN_UNIT, HIDDEN_NEURONS, HIDDEN_LAYERS, USE_CONCAT_INPUT, BATCH_SIZE, TRAINING_EPOCHS, USE_L2_LOSS, L2_FACTOR, USE_DROPOUT, DROPOUT_PROB, USE_CONSTANT_LIMIT, CONSTANT_PREDICTION_LIMIT, TRAINABLE_MATRIX, PRE_TRAINED_MATRIX, USE_PRETRAINED, VALIDATION_DATA, TRAINING_DATA, TESTING_DATA] From f46c8ce7e1688ccfcd735212111c2d6554e3bd57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20H=C3=A5kansson?= Date: Wed, 5 Apr 2017 13:40:25 +0200 Subject: [PATCH 13/27] Adds a missing variable name change --- model/model_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/model_builder.py b/model/model_builder.py index 02a5e5b..1121a33 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -101,7 +101,7 @@ def add_layer(self, number_of_neurons): if not self.added_layers: self.added_layers = True weights = tf.Variable(tf.random_normal( - [self._model.lstm_neurons + + [self._model.rnn_neurons + (1 if self._model.use_concat_input else 0), number_of_neurons], stddev=0.35, From 634d79b864b931c30e50c5e3fd52dfeaa3c7d9b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20H=C3=A5kansson?= Date: Wed, 5 Apr 2017 17:47:13 +0200 Subject: [PATCH 14/27] Changes default dataset in config template --- config.template.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config.template.yaml b/config.template.yaml index 1199a43..12d9cb9 100644 --- a/config.template.yaml +++ b/config.template.yaml @@ -8,9 +8,9 @@ network: vocabulary_size: 10000 user_count: 6 max_title_length: 30 - validation_data: 'validation_data_top_n_single.csv' - training_data: 'training_data_top_n_single.csv' - testing_data: 'testing_data_top_n_single.csv' + validation_data: 'validation_data_top_5_subreddit_allvotes.csv' + training_data: 'training_data_top_5_subreddit_allvotes.csv' + testing_data: 'testing_data_top_5_subreddit_allvotes.csv' # Embedding matrix configs: embedding_size: 150 # Make sure to match pretrained matrix dimensions trainable_matrix: true From d73a5c48bacb30c4e8d192cb275fd0b0be1c35b3 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Mon, 3 Apr 2017 13:28:45 +0200 Subject: [PATCH 15/27] merge --- config.template.yaml | 1 + main.py | 2 ++ model/model.py | 46 ++++++++++++++++++++++++++++++++++++++++ model/util/csv_reader.py | 2 +- model/util/data.py | 5 +++++ 5 files changed, 55 insertions(+), 1 deletion(-) diff --git a/config.template.yaml b/config.template.yaml index 12d9cb9..d70c202 100644 --- a/config.template.yaml +++ b/config.template.yaml @@ -25,6 +25,7 @@ network: hidden_layers: 0 hidden_neurons: 300 use_concat_input: false + pre_train_subreddit: false # Regularisation configs: use_l2_loss: false l2_factor: 0.01 diff --git a/main.py b/main.py index dd048e3..d2c06d6 100644 --- a/main.py +++ b/main.py @@ -39,6 +39,8 @@ def main(): with tf.Session() as sess: builder = ModelBuilder(config_file, sess) network_model = builder.build() + if config_file["pre-train-subreddit"]: + network_model.pre_train() network_model.train() network_model.close_writers() tf.reset_default_graph() diff --git a/model/model.py b/model/model.py index c028882..83b2758 100644 --- a/model/model.py +++ b/model/model.py @@ -232,6 +232,52 @@ def train(self): epoch_top=self.epoch_top, prec_valid=self.prec_valid, prec_train=self.prec_train, recall_valid=self.recall_valid, recall_train=self.recall_train) + def pre_train(self): + """ Pre-trains the model on the pre-training dataset """ + print("Starting pre-training...") + + old_epoch = 0 + + if self.use_pretrained: + self._session.run(self.embedding_init, feed_dict={ + self.embedding_placeholder: + self.data.embedding_matrix}) + + tmp_data, tmp_labels = self.data.get_training() + tmp_size = self.data.train_size + + self.data.train_data = self.data.pre_train_data + self.data.train_labels = self.data.pre_train_data + self.data.train_size = self.data.pre_train_size + + # Train for a specified amount of epochs + for i in self.data.for_n_train_epochs(self.training_epochs, + self.batch_size): + # Debug print out + epoch = self.data.completed_training_epochs + training_error = self.train_batch() + # validation_error = self.validate_batch() + + # Don't validate so often + if i % (self.data.train_size // self.batch_size // 10) == 0 and i: + done = self.data.percent_of_epoch + print( + "Validation error: {:f} | Training error: {:f} | Done: {:.0%}" + .format(0, training_error, done)) + + # Do a full evaluation once an epoch is complete + if epoch != old_epoch: + self._session.run(self.epoch.assign_add(1)) + print("Epoch complete...old ", old_epoch) + self.save_checkpoint() + # self.validate() + old_epoch = epoch + + self.data.train_data = tmp_data + self.data.train_labels = tmp_labels + self.data.train_size = tmp_size + self.data.completed_training_epochs = 0 + def train_batch(self): """ Trains for one batch and returns cross entropy error """ with tf.device("/cpu:0"): diff --git a/model/util/csv_reader.py b/model/util/csv_reader.py index c5b7526..9913c68 100644 --- a/model/util/csv_reader.py +++ b/model/util/csv_reader.py @@ -34,7 +34,7 @@ class Dataenum(Enum): TESTING = "testing_data" TRAINING = "training_data" VALIDATION = "validation_data" - + PRE_TRAINING = "pre_training_data" class CsvReader: def __init__(self, netcfg): diff --git a/model/util/data.py b/model/util/data.py index 4d9ac54..b50fb31 100644 --- a/model/util/data.py +++ b/model/util/data.py @@ -56,6 +56,11 @@ def __init__(self, networkconfig): def _read_data(self): """ Reads all the data from specified path """ + logging.debug("Reading pre-training data...") + + self.pre_train_data, self.pre_train_labels = self.reader.get_data(Dataenum.PRE_TRAINING) + self.pre_train_size = len(self.pre_train_data) + logging.debug("Reading training data...") self.train_data, self.train_subreddits, self.train_labels = \ From 468d07dacd3855707937b88777cc46966ec5c086 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Tue, 4 Apr 2017 10:55:16 +0200 Subject: [PATCH 16/27] added method to add secondary output layer --- model/model_builder.py | 56 +++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/model/model_builder.py b/model/model_builder.py index 1121a33..e2e196e 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -52,6 +52,10 @@ def add_input_layer(self): tf.placeholder(tf.float64, [None, self._model.user_count], name="target") + self._model.sec_target = \ + tf.placeholder(tf.float64, + [None, self._model.data.subreddit_count], + name="sec_target") self._model.keep_prob = tf.placeholder(tf.float64, name="keep_prob") @@ -184,6 +188,49 @@ def add_output_layer(self): return self + def add_secondary_output(self): + """Adds a layer that can be used to train the network on data that is + labeled in a different way than the final data""" + # Output layer + # Feed the output of the previous layer to a sigmoid layer + sigmoid_weights = tf.Variable(tf.random_normal( + [self._model.latest_layer.get_shape()[1].value, + self._model.subreddit_count], + stddev=0.35, + dtype=tf.float64), + name="secondary_output_weights") + + sigmoid_bias = tf.Variable(tf.random_normal([self._model.subreddit_count], + stddev=0.35, + dtype=tf.float64), + name="secondary_output_biases") + + logits = tf.add(tf.matmul(self._model.latest_layer, sigmoid_weights), + sigmoid_bias) + # Training + + # Defne error function + error = tf.nn.sigmoid_cross_entropy_with_logits( + labels=self._model.sec_target, + logits=logits) + + if self._model.use_l2_loss: + cross_entropy = \ + tf.reduce_mean(tf.add( + tf.add(error, + tf.multiply(self._model.l2_factor, + tf.nn.l2_loss(sigmoid_weights))), + tf.add(tf.multiply(self._model.l2_factor, + tf.nn.l2_loss(sigmoid_bias)), + tf.multiply(self._model.l2_factor, + self._model.l2_term)))) + else: + cross_entropy = tf.reduce_mean(error) + + self._model.pre_train_op = tf.train.AdamOptimizer( + self._model.learning_rate).minimize(cross_entropy) + return self + def add_precision_operations(self): """Adds precision operation and tensorboard operations""" # Determine which prediction function to use. Casts a tensor to @@ -273,15 +320,6 @@ def add_precision_operations(self): def build(self): """Adds saver and init operation and returns the model""" - self.add_input_layer() - - # Add a number of hidden layers - for _ in range(self._model.hidden_layers): - self.add_layer(self._model.hidden_neurons) - - self.add_output_layer() - - self.add_precision_operations() self._model.init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) From 2e957a375bfee31df6d981ed0c2b99d0def405b9 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Tue, 4 Apr 2017 10:58:53 +0200 Subject: [PATCH 17/27] implemented pre-training on subreddits --- main.py | 16 ++++++- model/model.py | 114 +++++++++++++++++++------------------------------ 2 files changed, 57 insertions(+), 73 deletions(-) diff --git a/main.py b/main.py index d2c06d6..68694b2 100644 --- a/main.py +++ b/main.py @@ -23,6 +23,7 @@ # ============================================================================== import argparse import tensorflow as tf +from definitions import * from model.util.networkconfig import yamlconfig as networkconfig from model.model_builder import ModelBuilder @@ -38,9 +39,20 @@ def main(): config_file = networkconfig[conf] with tf.Session() as sess: builder = ModelBuilder(config_file, sess) + builder.add_input_layer() + + # Add a number of hidden layers + for _ in range(config_file[HIDDEN_LAYERS]): + builder.add_layer(config_file[HIDDEN_NEURONS]) + + builder.add_output_layer()\ + .add_secondary_output()\ + .add_precision_operations() + network_model = builder.build() - if config_file["pre-train-subreddit"]: - network_model.pre_train() + if config_file[USE_PRETRAINED_NET]: + network_model.train(USE_PRETRAINED_NET) + network_model.data.completed_training_epochs = 0 network_model.train() network_model.close_writers() tf.reset_default_graph() diff --git a/model/model.py b/model/model.py index 83b2758..18f8a0a 100644 --- a/model/model.py +++ b/model/model.py @@ -68,13 +68,17 @@ def __init__(self, config, session): self.use_constant_limit = config[USE_CONSTANT_LIMIT] self.constant_prediction_limit = config[CONSTANT_PREDICTION_LIMIT] self.use_concat_input = config[USE_CONCAT_INPUT] + self.use_pretrained_net = config[USE_PRETRAINED_NET] + self.subreddit_count = 0 # Will be set in build_graph self.input = None self.subreddit_input = None self.target = None + self.sec_target = None self.sigmoid = None self.train_op = None + self.pre_train_op = None self.error = None self.init_op = None self.saver = None @@ -106,6 +110,7 @@ def __init__(self, config, session): with tf.device("/cpu:0"): self.data = data.Data(config) + self.subreddit_count = self.data.subreddit_count if self.use_pretrained: self.vocabulary_size = len(self.data.embedding_matrix) @@ -185,7 +190,7 @@ def validate_batch(self): # TODO funktionen gör alldeles för mycket, # dela upp utskrift, beräkning och träning - def train(self): + def train(self, use_pretrained_net=False): """ Trains the model on the dataset """ print("Starting training...") @@ -201,29 +206,33 @@ def train(self): old_epoch = 0 - if self.epoch.eval(self._session) == 0: + if self.epoch.eval(self._session) == 0 and not use_pretrained_net: self.validate() # Train for a specified amount of epochs - for i in self.data.for_n_train_epochs(self.training_epochs, - self.batch_size): + for i in self.data.for_n_train_epochs(self.batch_size, self.training_epochs): # Debug print out epoch = self.data.completed_training_epochs - training_error = self.train_batch() - validation_error = self.validate_batch() - # Don't validate so often - if i % (self.data.train_size // self.batch_size // 10) == 0 and i: - done = self.data.percent_of_epoch - print("Validation error: {:f} | Training error: {:f} | Done: {:.0%}" - .format(validation_error, training_error, done)) + if not use_pretrained_net: + training_error = self.train_batch() + validation_error = self.validate_batch() + + # Don't validate so often + if i % (self.data.train_size // self.batch_size // 10) == 0 and i: + done = self.data.percent_of_epoch + print("Validation error: {:f} | Training error: {:f} | Done: {:.0%}" + .format(validation_error, training_error, done)) + else: + self.train_batch(True) # Do a full evaluation once an epoch is complete if epoch != old_epoch: self._session.run(self.epoch.assign_add(1)) print("Epoch complete...old ", old_epoch) self.save_checkpoint() - self.validate() + if not self.use_pretrained_net: + self.validate() old_epoch = epoch # Save model when done training @@ -232,67 +241,30 @@ def train(self): epoch_top=self.epoch_top, prec_valid=self.prec_valid, prec_train=self.prec_train, recall_valid=self.recall_valid, recall_train=self.recall_train) - def pre_train(self): - """ Pre-trains the model on the pre-training dataset """ - print("Starting pre-training...") - - old_epoch = 0 - - if self.use_pretrained: - self._session.run(self.embedding_init, feed_dict={ - self.embedding_placeholder: - self.data.embedding_matrix}) - - tmp_data, tmp_labels = self.data.get_training() - tmp_size = self.data.train_size - - self.data.train_data = self.data.pre_train_data - self.data.train_labels = self.data.pre_train_data - self.data.train_size = self.data.pre_train_size - - # Train for a specified amount of epochs - for i in self.data.for_n_train_epochs(self.training_epochs, - self.batch_size): - # Debug print out - epoch = self.data.completed_training_epochs - training_error = self.train_batch() - # validation_error = self.validate_batch() - - # Don't validate so often - if i % (self.data.train_size // self.batch_size // 10) == 0 and i: - done = self.data.percent_of_epoch - print( - "Validation error: {:f} | Training error: {:f} | Done: {:.0%}" - .format(0, training_error, done)) - - # Do a full evaluation once an epoch is complete - if epoch != old_epoch: - self._session.run(self.epoch.assign_add(1)) - print("Epoch complete...old ", old_epoch) - self.save_checkpoint() - # self.validate() - old_epoch = epoch - - self.data.train_data = tmp_data - self.data.train_labels = tmp_labels - self.data.train_size = tmp_size - self.data.completed_training_epochs = 0 - - def train_batch(self): + def train_batch(self, pre_train_net=False): """ Trains for one batch and returns cross entropy error """ with tf.device("/cpu:0"): - batch_input, batch_sub, batch_label = \ - self.data.next_train_batch() - - self._session.run(self.train_op, - {self.input: batch_input, - self.subreddit_input: batch_sub, - self.target: batch_label}) - - return self._session.run(self.error, - feed_dict={self.input: batch_input, - self.subreddit_input: batch_sub, - self.target: batch_label}) + if not pre_train_net: + batch_input, batch_sub, batch_label = \ + self.data.next_train_batch() + else: + batch_input, batch_label = \ + self.data.next_pre_train_batch() + + if pre_train_net: + self._session.run(self.pre_train_op, + {self.input: batch_input, + self.sec_target: batch_label}) + else: + self._session.run(self.train_op, + {self.input: batch_input, + self.subreddit_input: batch_sub, + self.target: batch_label}) + + return self._session.run(self.error, + feed_dict={self.input: batch_input, + self.subreddit_input: batch_sub, + self.target: batch_label}) def close_writers(self): """ Close tensorboard writers """ self.train_writer.close() From 796a81ded518244e0fa01162ad4f1d9358b34717 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Tue, 4 Apr 2017 10:59:30 +0200 Subject: [PATCH 18/27] create methods for handling pre-training data --- definitions.py | 1 + model/util/data.py | 45 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/definitions.py b/definitions.py index efdcf9e..6ff5dac 100644 --- a/definitions.py +++ b/definitions.py @@ -56,6 +56,7 @@ TRAINABLE_MATRIX = 'trainable_matrix' PRE_TRAINED_MATRIX = 'pre_trained_matrix' USE_PRETRAINED = 'use_pretrained' +USE_PRETRAINED_NET = 'pre_train_subreddit' VALIDATION_DATA = 'validation_data' TRAINING_DATA = 'training_data' TESTING_DATA = 'testing_data' diff --git a/model/util/data.py b/model/util/data.py index b50fb31..7300967 100644 --- a/model/util/data.py +++ b/model/util/data.py @@ -35,8 +35,11 @@ def __init__(self, networkconfig): self._current_train_index = 0 self._current_valid_index = 0 self._current_test_index = 0 + self._current_pre_train_index = 0 self.completed_training_epochs = 0 + self.completed_pre_training_epochs = 0 self.percent_of_epoch = 0.0 + self.subreddit_count = 0 self.title_length = networkconfig['max_title_length'] self.batch_size = self.netcfg['batch_size'] self.reader = CsvReader(networkconfig) @@ -58,8 +61,8 @@ def _read_data(self): """ Reads all the data from specified path """ logging.debug("Reading pre-training data...") - self.pre_train_data, self.pre_train_labels = self.reader.get_data(Dataenum.PRE_TRAINING) - self.pre_train_size = len(self.pre_train_data) + # self.pre_train_data, self.pre_train_labels = self.reader.get_data(Dataenum.TRAINING, label_column=1) + # self.pre_train_size = len(self.pre_train_data) logging.debug("Reading training data...") @@ -97,6 +100,7 @@ def _build_dict(self): subreddits = " ".join(self.train_subreddits).split() self.subreddit_dict = helper.build_subreddit_dict(subreddits) + self.subreddit_count = len(self.subreddit_dict) def next_train_batch(self, batch_size=None): """ Get the next batch of training data """ @@ -136,6 +140,39 @@ def next_train_batch(self, batch_size=None): self.percent_of_epoch = self._current_train_index / self.train_size return batch_x, batch_sub, batch_y + def next_pre_train_batch(self, batch_size=None): + """ Get the next batch of training data """ + batch_size = batch_size or self.batch_size + batch_x = [] + batch_y = [] + for _ in range(0, batch_size): + sentence = self.train_data[self._current_train_index] + label = self.train_labels[self._current_train_index] + self._current_pre_train_index += 1 + # Support multiple epochs + if self._current_pre_train_index >= self.train_size: + self._current_pre_train_index = 0 + self.completed_pre_training_epochs += 1 + self.percent_of_epoch = 0.0 + # TODO Detta ska inte ligga i funktionen som generar ny data + + # Turn sentences and labels into vector representations + sentence_vec, present, absent = \ + helper.get_indicies(sentence, + self.word_dict, + self.max_title_length) + self.train_present += present + self.train_absent += absent + + label_vec = helper.label_vector(label, + self.subreddit_dict, + self.subreddit_count) + batch_x.append(sentence_vec) + batch_y.append(label_vec) + + self.percent_of_epoch = self._current_train_index / self.train_size + return batch_x, batch_y + def get_validation(self): """ Get the whole validation set in a vectorized form """ old_ind = self._current_valid_index @@ -220,10 +257,10 @@ def next_test_batch(self, batch_size=None): batch_y.append(label_vec) return batch_x, batch_sub, batch_y - def for_n_train_epochs(self, num_epochs=1, batch_size=25): + def for_n_train_epochs(self, batch_size=25, num_epochs=1): # TODO Ta bort parameterar """ Calculates how many training iterations to do for num_epochs - number of epochs with a batch size of batch_size """ + number of epochs with a batch size of batch_size""" return range((self.train_size * num_epochs) // batch_size) def get_training(self): From 81c149d4fd035012eb1b00c30dbe2b2f661f9039 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Tue, 4 Apr 2017 14:10:06 +0200 Subject: [PATCH 19/27] non-working commit to allow for debugg help. --- main.py | 1 - model/model.py | 11 ++++------- model/model_builder.py | 6 ++++++ model/util/data.py | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index 68694b2..78e9255 100644 --- a/main.py +++ b/main.py @@ -52,7 +52,6 @@ def main(): network_model = builder.build() if config_file[USE_PRETRAINED_NET]: network_model.train(USE_PRETRAINED_NET) - network_model.data.completed_training_epochs = 0 network_model.train() network_model.close_writers() tf.reset_default_graph() diff --git a/model/model.py b/model/model.py index 18f8a0a..726e8d5 100644 --- a/model/model.py +++ b/model/model.py @@ -194,15 +194,12 @@ def train(self, use_pretrained_net=False): """ Trains the model on the dataset """ print("Starting training...") - if self.use_pretrained: + if self.use_pretrained and \ + (self.use_pretrained_net and use_pretrained_net) or \ + (not self.use_pretrained_net and not use_pretrained_net): self._session.run(self.embedding_init, feed_dict={self.embedding_placeholder: self.data.embedding_matrix}) - self.train_writer = \ - tf.summary.FileWriter(self.logging_dir + '/' + TENSOR_DIR_TRAIN, - self._session.graph) - self.valid_writer = \ - tf.summary.FileWriter(self.logging_dir + '/' + TENSOR_DIR_VALID) old_epoch = 0 @@ -210,7 +207,7 @@ def train(self, use_pretrained_net=False): self.validate() # Train for a specified amount of epochs - for i in self.data.for_n_train_epochs(self.batch_size, self.training_epochs): + for i in self.data.for_n_train_epochs(self.training_epochs, self.batch_size): # Debug print out epoch = self.data.completed_training_epochs diff --git a/model/model_builder.py b/model/model_builder.py index e2e196e..4f96ef3 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -24,6 +24,7 @@ import tensorflow as tf from model.model import Model +from definitions import * class ModelBuilder(object): """A class following the builder pattern to create a model""" @@ -321,6 +322,11 @@ def add_precision_operations(self): def build(self): """Adds saver and init operation and returns the model""" + self._model.train_writer = \ + tf.summary.FileWriter(self._model.logging_dir + '/' + TENSOR_DIR_TRAIN, + self._model._session.graph) + self._model.valid_writer = \ + tf.summary.FileWriter(self._model.logging_dir + '/' + TENSOR_DIR_VALID) self._model.init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) self._model.saver = tf.train.Saver() diff --git a/model/util/data.py b/model/util/data.py index 7300967..3da2866 100644 --- a/model/util/data.py +++ b/model/util/data.py @@ -257,7 +257,7 @@ def next_test_batch(self, batch_size=None): batch_y.append(label_vec) return batch_x, batch_sub, batch_y - def for_n_train_epochs(self, batch_size=25, num_epochs=1): + def for_n_train_epochs(self, num_epochs=1, batch_size=25): # TODO Ta bort parameterar """ Calculates how many training iterations to do for num_epochs number of epochs with a batch size of batch_size""" From f5c49ed5ea037808e0354d3de22ba80a0695fed4 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 11:09:52 +0200 Subject: [PATCH 20/27] changed which variable was used to check if pre-training --- model/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/model.py b/model/model.py index 726e8d5..4316c82 100644 --- a/model/model.py +++ b/model/model.py @@ -228,7 +228,7 @@ def train(self, use_pretrained_net=False): self._session.run(self.epoch.assign_add(1)) print("Epoch complete...old ", old_epoch) self.save_checkpoint() - if not self.use_pretrained_net: + if not use_pretrained_net: self.validate() old_epoch = epoch From 8c5cb78929a2555d5bd25ff139239ad4aa2f200e Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 11:12:18 +0200 Subject: [PATCH 21/27] only add sec_output when pre-train and add an extra layer after --- main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 78e9255..2b63eaa 100644 --- a/main.py +++ b/main.py @@ -45,8 +45,11 @@ def main(): for _ in range(config_file[HIDDEN_LAYERS]): builder.add_layer(config_file[HIDDEN_NEURONS]) + if builder._model.use_pretrained_net: + builder.add_secondary_output() + builder.add_layer(config_file[HIDDEN_NEURONS]) + builder.add_output_layer()\ - .add_secondary_output()\ .add_precision_operations() network_model = builder.build() From 7383a0b4fd6db1c9133362520ece9acaa040b2d5 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 17:46:21 +0200 Subject: [PATCH 22/27] cleaned main.py by adding it to builder --- main.py | 14 +------------- model/model_builder.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/main.py b/main.py index 2b63eaa..6780e31 100644 --- a/main.py +++ b/main.py @@ -39,20 +39,8 @@ def main(): config_file = networkconfig[conf] with tf.Session() as sess: builder = ModelBuilder(config_file, sess) - builder.add_input_layer() - # Add a number of hidden layers - for _ in range(config_file[HIDDEN_LAYERS]): - builder.add_layer(config_file[HIDDEN_NEURONS]) - - if builder._model.use_pretrained_net: - builder.add_secondary_output() - builder.add_layer(config_file[HIDDEN_NEURONS]) - - builder.add_output_layer()\ - .add_precision_operations() - - network_model = builder.build() + network_model = builder.add_as_conf().build() if config_file[USE_PRETRAINED_NET]: network_model.train(USE_PRETRAINED_NET) network_model.train() diff --git a/model/model_builder.py b/model/model_builder.py index 4f96ef3..bcb411b 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -332,3 +332,17 @@ def build(self): self._model.saver = tf.train.Saver() self._model.load_checkpoint() return self._model + + def add_as_conf(self): + self.add_input_layer() + + # Add a number of hidden layers + for _ in range(self._model.hidden_layers): + self.add_layer(self._model.hidden_neurons) + + if self._model.use_pretrained_net: + self.add_secondary_output() + + self.add_output_layer() \ + .add_precision_operations() + return self From 157987d005a011c757462faaf67121ae47fbadb1 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 17:50:16 +0200 Subject: [PATCH 23/27] sec_output uses softmax since a title only have one subreddit --- model/model_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/model_builder.py b/model/model_builder.py index bcb411b..895f983 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -211,7 +211,7 @@ def add_secondary_output(self): # Training # Defne error function - error = tf.nn.sigmoid_cross_entropy_with_logits( + error = tf.nn.softmax_cross_entropy_with_logits( labels=self._model.sec_target, logits=logits) From 90086abcd19526e7ab62727a0d1ec7efdf6bde99 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Wed, 5 Apr 2017 18:15:10 +0200 Subject: [PATCH 24/27] refactored add output method to one method --- model/model_builder.py | 77 ++++++++++++------------------------------ 1 file changed, 22 insertions(+), 55 deletions(-) diff --git a/model/model_builder.py b/model/model_builder.py index 895f983..f6010bb 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -144,77 +144,38 @@ def add_layer(self, number_of_neurons): return self - def add_output_layer(self): + def add_output_layer(self, output_size, secondary_output=False): """Adds an output layer, including error and optimisation functions. After this method no new layers should be added.""" # Output layer # Feed the output of the previous layer to a sigmoid layer sigmoid_weights = tf.Variable(tf.random_normal( - [self._model.latest_layer.get_shape()[1].value, self._model.user_count], + [self._model.latest_layer.get_shape()[1].value, output_size], stddev=0.35, dtype=tf.float64), name="output_weights") - sigmoid_bias = tf.Variable(tf.random_normal([self._model.user_count], + sigmoid_bias = tf.Variable(tf.random_normal([output_size], stddev=0.35, dtype=tf.float64), name="output_biases") logits = tf.add(tf.matmul(self._model.latest_layer, sigmoid_weights), sigmoid_bias) - self._model.sigmoid = tf.nn.sigmoid(logits) - # Training - - # Defne error function - error = tf.nn.sigmoid_cross_entropy_with_logits(labels=self._model.target, - logits=logits) - - if self._model.use_l2_loss: - cross_entropy = \ - tf.reduce_mean(tf.add( - tf.add(error, - tf.multiply(self._model.l2_factor, - tf.nn.l2_loss(sigmoid_weights))), - tf.add(tf.multiply(self._model.l2_factor, - tf.nn.l2_loss(sigmoid_bias)), - tf.multiply(self._model.l2_factor, - self._model.l2_term)))) + if secondary_output: + error = tf.nn.softmax_cross_entropy_with_logits( + labels=self._model.sec_target, + logits=logits) else: - cross_entropy = tf.reduce_mean(error) - - self._model.error = cross_entropy - self._model.train_op = tf.train.AdamOptimizer( - self._model.learning_rate).minimize(cross_entropy) + self._model.sigmoid = tf.nn.sigmoid(logits) + # Defne error function + error = tf.nn.sigmoid_cross_entropy_with_logits( + labels=self._model.target, + logits=logits) - return self - - def add_secondary_output(self): - """Adds a layer that can be used to train the network on data that is - labeled in a different way than the final data""" - # Output layer - # Feed the output of the previous layer to a sigmoid layer - sigmoid_weights = tf.Variable(tf.random_normal( - [self._model.latest_layer.get_shape()[1].value, - self._model.subreddit_count], - stddev=0.35, - dtype=tf.float64), - name="secondary_output_weights") - - sigmoid_bias = tf.Variable(tf.random_normal([self._model.subreddit_count], - stddev=0.35, - dtype=tf.float64), - name="secondary_output_biases") - - logits = tf.add(tf.matmul(self._model.latest_layer, sigmoid_weights), - sigmoid_bias) # Training - # Defne error function - error = tf.nn.softmax_cross_entropy_with_logits( - labels=self._model.sec_target, - logits=logits) - if self._model.use_l2_loss: cross_entropy = \ tf.reduce_mean(tf.add( @@ -228,8 +189,14 @@ def add_secondary_output(self): else: cross_entropy = tf.reduce_mean(error) - self._model.pre_train_op = tf.train.AdamOptimizer( - self._model.learning_rate).minimize(cross_entropy) + if secondary_output: + self._model.pre_train_op = tf.train.AdamOptimizer( + self._model.learning_rate).minimize(cross_entropy) + else: + self._model.error = cross_entropy + self._model.train_op = tf.train.AdamOptimizer( + self._model.learning_rate).minimize(cross_entropy) + return self def add_precision_operations(self): @@ -341,8 +308,8 @@ def add_as_conf(self): self.add_layer(self._model.hidden_neurons) if self._model.use_pretrained_net: - self.add_secondary_output() + self.add_output_layer(self._model.subreddit_count, secondary_output=True) - self.add_output_layer() \ + self.add_output_layer(self._model.user_count) \ .add_precision_operations() return self From 65cf80c4510392016e79dcac3b49adbbacc7c76b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20H=C3=A5kansson?= Date: Wed, 5 Apr 2017 18:48:29 +0200 Subject: [PATCH 25/27] Refactors away redundant function in model builder --- main.py | 2 +- model/model.py | 7 ++++--- model/model_builder.py | 33 ++++++++++++++++++--------------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/main.py b/main.py index 6780e31..030cb35 100644 --- a/main.py +++ b/main.py @@ -40,7 +40,7 @@ def main(): with tf.Session() as sess: builder = ModelBuilder(config_file, sess) - network_model = builder.add_as_conf().build() + network_model = builder.build() if config_file[USE_PRETRAINED_NET]: network_model.train(USE_PRETRAINED_NET) network_model.train() diff --git a/model/model.py b/model/model.py index 4316c82..78beb16 100644 --- a/model/model.py +++ b/model/model.py @@ -188,11 +188,12 @@ def validate_batch(self): self.subreddit_input: batch_sub, self.target: batch_label}) - # TODO funktionen gör alldeles för mycket, - # dela upp utskrift, beräkning och träning def train(self, use_pretrained_net=False): """ Trains the model on the dataset """ - print("Starting training...") + if use_pretrained_net: + print("Pre-training on subreddits...") + else: + print("Starting training...") if self.use_pretrained and \ (self.use_pretrained_net and use_pretrained_net) or \ diff --git a/model/model_builder.py b/model/model_builder.py index f6010bb..bf9f19d 100644 --- a/model/model_builder.py +++ b/model/model_builder.py @@ -165,8 +165,8 @@ def add_output_layer(self, output_size, secondary_output=False): if secondary_output: error = tf.nn.softmax_cross_entropy_with_logits( - labels=self._model.sec_target, - logits=logits) + labels=self._model.sec_target, + logits=logits) else: self._model.sigmoid = tf.nn.sigmoid(logits) # Defne error function @@ -289,27 +289,30 @@ def add_precision_operations(self): def build(self): """Adds saver and init operation and returns the model""" - self._model.train_writer = \ - tf.summary.FileWriter(self._model.logging_dir + '/' + TENSOR_DIR_TRAIN, - self._model._session.graph) - self._model.valid_writer = \ - tf.summary.FileWriter(self._model.logging_dir + '/' + TENSOR_DIR_VALID) - self._model.init_op = tf.group(tf.global_variables_initializer(), - tf.local_variables_initializer()) - self._model.saver = tf.train.Saver() - self._model.load_checkpoint() - return self._model - - def add_as_conf(self): + # Add input layer self.add_input_layer() # Add a number of hidden layers for _ in range(self._model.hidden_layers): self.add_layer(self._model.hidden_neurons) + # Add output layer for pretraining, if used if self._model.use_pretrained_net: self.add_output_layer(self._model.subreddit_count, secondary_output=True) + # Add output layer for users self.add_output_layer(self._model.user_count) \ .add_precision_operations() - return self + + # Initialize + self._model.train_writer = \ + tf.summary.FileWriter(self._model.logging_dir + '/' + TENSOR_DIR_TRAIN, + self._model._session.graph) + self._model.valid_writer = \ + tf.summary.FileWriter(self._model.logging_dir + '/' + TENSOR_DIR_VALID) + + self._model.init_op = tf.group(tf.global_variables_initializer(), + tf.local_variables_initializer()) + self._model.saver = tf.train.Saver() + self._model.load_checkpoint() + return self._model From fbe203f58d90302929cabdd85c2217daf938167d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20H=C3=A5kansson?= Date: Wed, 5 Apr 2017 19:24:57 +0200 Subject: [PATCH 26/27] Removes un-used constant --- model/util/csv_reader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/model/util/csv_reader.py b/model/util/csv_reader.py index 9913c68..17c8357 100644 --- a/model/util/csv_reader.py +++ b/model/util/csv_reader.py @@ -34,7 +34,6 @@ class Dataenum(Enum): TESTING = "testing_data" TRAINING = "training_data" VALIDATION = "validation_data" - PRE_TRAINING = "pre_training_data" class CsvReader: def __init__(self, netcfg): From ac40573c0b292970cccf0a5cf8336deec12e0b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20H=C3=A5kansson?= Date: Wed, 5 Apr 2017 19:43:00 +0200 Subject: [PATCH 27/27] Removes unused epoch counting for pre-training --- model/model.py | 3 +-- model/util/data.py | 9 --------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/model/model.py b/model/model.py index 78beb16..a0f3494 100644 --- a/model/model.py +++ b/model/model.py @@ -229,8 +229,7 @@ def train(self, use_pretrained_net=False): self._session.run(self.epoch.assign_add(1)) print("Epoch complete...old ", old_epoch) self.save_checkpoint() - if not use_pretrained_net: - self.validate() + self.validate() old_epoch = epoch # Save model when done training diff --git a/model/util/data.py b/model/util/data.py index 3da2866..7672e32 100644 --- a/model/util/data.py +++ b/model/util/data.py @@ -37,7 +37,6 @@ def __init__(self, networkconfig): self._current_test_index = 0 self._current_pre_train_index = 0 self.completed_training_epochs = 0 - self.completed_pre_training_epochs = 0 self.percent_of_epoch = 0.0 self.subreddit_count = 0 self.title_length = networkconfig['max_title_length'] @@ -59,10 +58,6 @@ def __init__(self, networkconfig): def _read_data(self): """ Reads all the data from specified path """ - logging.debug("Reading pre-training data...") - - # self.pre_train_data, self.pre_train_labels = self.reader.get_data(Dataenum.TRAINING, label_column=1) - # self.pre_train_size = len(self.pre_train_data) logging.debug("Reading training data...") @@ -152,9 +147,6 @@ def next_pre_train_batch(self, batch_size=None): # Support multiple epochs if self._current_pre_train_index >= self.train_size: self._current_pre_train_index = 0 - self.completed_pre_training_epochs += 1 - self.percent_of_epoch = 0.0 - # TODO Detta ska inte ligga i funktionen som generar ny data # Turn sentences and labels into vector representations sentence_vec, present, absent = \ @@ -170,7 +162,6 @@ def next_pre_train_batch(self, batch_size=None): batch_x.append(sentence_vec) batch_y.append(label_vec) - self.percent_of_epoch = self._current_train_index / self.train_size return batch_x, batch_y def get_validation(self):