diff --git a/.gitignore b/.gitignore index e43b0f9..4ac1591 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,8 @@ .DS_Store +data +*.bak +__init__.py +.ipynb_checkpoints +__pycache__ +logs/cats-train-full +lda \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8696ded --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.8 + +WORKDIR /api + +RUN python3 -m pip install --upgrade pip + +COPY requirements.txt . + +RUN pip3 install -r requirements.txt + +COPY . . + +EXPOSE 5000 + +CMD ["python3", "./api.py"] \ No newline at end of file diff --git a/README.md b/README.md index c720b2d..1096163 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repository contains code related to the paper “CATS: Customizable Abstractive Topic-based Summarization” published at Transactions of Information Systems (TOIS) journal, 2021. -The code has been developed using python 2.7, and Tensorflow 1.4. This implementation is based on code releases related to Pointer-Generator Networks [here](https://github.com/abisee/pointer-generator) and the TextSum project. +The code has been developed using Python 3, and v1 compatible TensorFlow 2 code. This implementation is based on code releases related to Pointer-Generator Networks [here](https://github.com/abisee/pointer-generator) and the TextSum project. #### Dataset @@ -14,13 +14,13 @@ In order to obtain the dataset, we encourage users to download and preprocess th #### Using Topic Information: -The LDA models used in our paper can be obtained from [here](https://drive.google.com/drive/folders/1M86uAM21Zx8Xn-W4TTi74t7nWCQr1f9v?usp=sharing). The current code release has been tested with the 150 topics pre-trained LDA model. You can make a reference to one of the provided LDA topic models in data.py in the TopicModel class. +The LDA models used in our paper can be obtained from [here](https://drive.google.com/drive/folders/1M86uAM21Zx8Xn-W4TTi74t7nWCQr1f9v?usp=sharing). The current code release has been tested with the 150 topics pre-trained LDA model. You can make a reference to one of the provided LDA topic models in data.py in the TopicModel class. By default, the LDA model is expected to be in a folder called `lda`, which contains `lda.model` and `dictionary.dic` available from the beforementioned URL. #### Train In order to train the model you may run: ``` -python run_summarization.py --mode=train --data_path=/path/to/chunked/train_* --vocab_path=/path/to/vocab --log_root=/path/to/a/log/directory --exp_name=myexperiment +python3 run_summarization.py --mode=train --data_path='data/chunked/train_*' --vocab_path='data/vocab' --log_root='logs' --exp_name=myexperiment ``` This will create a subdirectory of your specified log_root called myexperiment where all checkpoints will be saved. Then the model will start training using the train_*.bin files as training data. @@ -30,7 +30,7 @@ This will create a subdirectory of your specified log_root called myexperiment w As stated in the paper, no topic information were used at test time. In order to decode without topic information, we used the pointer-generator basic model code [here](https://github.com/abisee/pointer-generator). After downloading the code, you may decode using: ``` -python run_summarization.py --mode=decode --data_path=/path/to/chunked/val_* --vocab_path=/path/to/vocab --log_root=/path/to/a/log/directory --exp_name=myexperiment +python3 run_summarization.py --mode=decode --data_path='data/chunked/val_*' --vocab_path='data/vocab' --log_root='logs' --exp_name=myexperiment ``` Please note that one should run the above command using the same settings entered for the training job (plus any decode mode specific flags like beam_size). @@ -38,3 +38,21 @@ Please note that one should run the above command using the same settings entere This will repeatedly load random examples from your specified datafile and generate a summary using beam search. The results will be printed to screen. If you would like to run evaluation on the entire validation or test set and obtain ROUGE scores, set the flag single_pass=1. This will go through the entire dataset in the same order, writing the generated summaries to file, and then running evaluation using pyrouge. + +### CATS API +To use the API, build the Docker container using the following command: + +``` +docker build -t cats-api-container . +``` + +Then, run the Docker container with the following command: + +``` +docker run -d -p 5000:5000 cats-api-container +``` + +This should expose the container on port 5000. To summarize one input text, you can send a POST-request to `/summarize`, structuring the request as follows: + +- The request must have form-data as body +- The form-data object should include a key called `target`, with the value being the to-be-summarized text. \ No newline at end of file diff --git a/__init__.py b/__init__.py old mode 100755 new mode 100644 diff --git a/api.py b/api.py new file mode 100644 index 0000000..c198f96 --- /dev/null +++ b/api.py @@ -0,0 +1,47 @@ +from flask import Flask, request +from encode import write_to_bin +import uuid +import os +import shutil + +app = Flask(__name__) +app.config["DEBUG"] = False + +@app.route('/', methods=['GET']) +def home(): + return "

CATS: Customizable Abstractive Topic-based Summarization

This site is the interface of an API to interact with an advanced topic-aware summarization model.

" + +@app.route('/summarize', methods=['POST']) +def get_summary(): + # Define settings + TARGET_INPUT_FOLDER = 'requests' + TRAINED_MODEL_FOLDER = 'logs' + TRAINED_MODEL_EXPERIMENT = 'cats-train-full' + + # Generate other variables + request_uuid = request_uuid = str(uuid.uuid4()) + input_file_location = os.path.join(TARGET_INPUT_FOLDER, request_uuid, 'test_000.bin') + + # Get target text + input_text = request.form.get('target') + + # Write target to binary for decoding + write_to_bin(input_text, input_file_location) + + # Create summary + os.system(f"python3 run_summarization.py --mode=decode --data_path='{os.path.join(TARGET_INPUT_FOLDER, request_uuid)}/test_*' --vocab_path='data/vocab' --log_root='{TRAINED_MODEL_FOLDER}' --exp_name={TRAINED_MODEL_EXPERIMENT} --single_pass=True --single_input=True --decode_dir='{os.path.join('requests', request_uuid)}'") + + # Read the summary file + with open(os.path.join(TARGET_INPUT_FOLDER, request_uuid, 'decoded', '000000_decoded.txt')) as f: + summary = f.read().splitlines() + + # Delete the request directory + shutil.rmtree(os.path.join(TARGET_INPUT_FOLDER, request_uuid)) + + # Return the summary + return { + 'summary': summary + } + +if __name__ == '__main__': + app.run(host="0.0.0.0", port=5000) \ No newline at end of file diff --git a/attention_decoder.py b/attention_decoder.py old mode 100755 new mode 100644 index 44a6a11..0f843b2 --- a/attention_decoder.py +++ b/attention_decoder.py @@ -49,12 +49,12 @@ def attention_decoder(decoder_inputs, initial_state, encoder_states, enc_topicwo coverage: Coverage vector on the last step computed. None if use_coverage=False. """ with variable_scope.variable_scope("attention_decoder") as scope: - batch_size = encoder_states.get_shape()[0].value # if this line fails, it's because the batch size isn't defined - attn_size = encoder_states.get_shape()[2].value # if this line fails, it's because the attention length isn't defined + batch_size = encoder_states.get_shape()[0] # if this line fails, it's because the batch size isn't defined + attn_size = encoder_states.get_shape()[2] # if this line fails, it's because the attention length isn't defined #tf.Print(attn_dist, [attn_dist]) - print "-------------------------" - print "-------------------------" + print("-------------------------") + print("-------------------------") # Reshape encoder_states (need to insert a dim) encoder_states = tf.expand_dims(encoder_states, axis=2) # now is shape (batch_size, attn_len, 1, attn_size) @@ -101,7 +101,7 @@ def masked_attention(e): """Take softmax of e then apply enc_padding_mask and re-normalize""" attn_dist = nn_ops.softmax(e) # take softmax. shape (batch_size, attn_length) attn_dist *= enc_padding_mask # apply mask - masked_sums = tf.reduce_sum(attn_dist, axis=1) # shape (batch_size) + masked_sums = tf.reduce_sum(input_tensor=attn_dist, axis=1) # shape (batch_size) return attn_dist / tf.reshape(masked_sums, [-1, 1]) # re-normalize if use_coverage and coverage is not None: # non-first step of coverage @@ -146,13 +146,13 @@ def masked_attention(e): # Re-calculate the context vector from the previous step so that we can pass it through a linear layer with this step's input to get a modified version of the input context_vector, _, coverage = attention(initial_state, enc_topicwords_probs_batch, coverage) # in decode mode, this is what updates the coverage vector for i, inp in enumerate(decoder_inputs): - tf.logging.info("Adding attention_decoder timestep %i of %i", i, len(decoder_inputs)) + tf.compat.v1.logging.info("Adding attention_decoder timestep %i of %i", i, len(decoder_inputs)) if i > 0: variable_scope.get_variable_scope().reuse_variables() # Merge input and previous attentions into one vector x of the same size as inp input_size = inp.get_shape().with_rank(2)[1] - if input_size.value is None: + if input_size is None: raise ValueError("Could not infer input size from input: %s" % inp.name) x = linear([inp] + [context_vector], input_size, True) @@ -169,7 +169,7 @@ def masked_attention(e): # Calculate p_gen if pointer_gen: - with tf.variable_scope('calculate_pgen'): + with tf.compat.v1.variable_scope('calculate_pgen'): p_gen = linear([context_vector, state.c, state.h, x], 1, True) # Tensor shape (batch_size, 1) p_gen = tf.sigmoid(p_gen) p_gens.append(p_gen) @@ -227,14 +227,14 @@ def linear(args, output_size, bias, bias_start=0.0, scope=None): total_arg_size += shape[1] # Now the computation. - with tf.variable_scope(scope or "Linear"): - matrix = tf.get_variable("Matrix", [total_arg_size, output_size]) + with tf.compat.v1.variable_scope(scope or "Linear"): + matrix = tf.compat.v1.get_variable("Matrix", [total_arg_size, output_size]) if len(args) == 1: res = tf.matmul(args[0], matrix) else: res = tf.matmul(tf.concat(axis=1, values=args), matrix) if not bias: return res - bias_term = tf.get_variable( - "Bias", [output_size], initializer=tf.constant_initializer(bias_start)) + bias_term = tf.compat.v1.get_variable( + "Bias", [output_size], initializer=tf.compat.v1.constant_initializer(bias_start)) return res + bias_term diff --git a/batcher.py b/batcher.py old mode 100755 new mode 100644 index 32feca1..667ca13 --- a/batcher.py +++ b/batcher.py @@ -16,7 +16,7 @@ """This file contains code to process data into batches""" -import Queue +import queue from random import shuffle from threading import Thread import time @@ -199,7 +199,7 @@ def init_encoder_seq(self, example_list, hps): self.enc_topicwords_batch[i, :] = ex.article_topic_vector[:] self.enc_topicwords_probs_batch[i, :] = ex.article_topic_word_probs[:] self.enc_lens[i] = ex.enc_len - for j in xrange(ex.enc_len): + for j in range(ex.enc_len): self.enc_padding_mask[i][j] = 1 # For pointer-generator mode, need to store some extra info @@ -236,7 +236,7 @@ def init_decoder_seq(self, example_list, hps): for i, ex in enumerate(example_list): self.dec_batch[i, :] = ex.dec_input[:] self.target_batch[i, :] = ex.target[:] - for j in xrange(ex.dec_len): + for j in range(ex.dec_len): self.dec_padding_mask[i][j] = 1 def store_orig_strings(self, example_list): @@ -266,8 +266,8 @@ def __init__(self, data_path, vocab, hps, single_pass): self._single_pass = single_pass # Initialize a queue of Batches waiting to be used, and a queue of Examples waiting to be batched - self._batch_queue = Queue.Queue(self.BATCH_QUEUE_MAX) - self._example_queue = Queue.Queue(self.BATCH_QUEUE_MAX * self._hps.batch_size) + self._batch_queue = queue.Queue(self.BATCH_QUEUE_MAX) + self._example_queue = queue.Queue(self.BATCH_QUEUE_MAX * self._hps.batch_size) # Different settings depending on whether we're in single_pass mode or not if single_pass: @@ -282,12 +282,12 @@ def __init__(self, data_path, vocab, hps, single_pass): # Start the threads that load the queues self._example_q_threads = [] - for _ in xrange(self._num_example_q_threads): + for _ in range(self._num_example_q_threads): self._example_q_threads.append(Thread(target=self.fill_example_queue)) self._example_q_threads[-1].daemon = True self._example_q_threads[-1].start() self._batch_q_threads = [] - for _ in xrange(self._num_batch_q_threads): + for _ in range(self._num_batch_q_threads): self._batch_q_threads.append(Thread(target=self.fill_batch_queue)) self._batch_q_threads[-1].daemon = True self._batch_q_threads[-1].start() @@ -309,9 +309,9 @@ def next_batch(self): """ # If the batch queue is empty, print a warning if self._batch_queue.qsize() == 0: - tf.logging.warning('Bucket input queue is empty when calling next_batch. Bucket queue size: %i, Input queue size: %i', self._batch_queue.qsize(), self._example_queue.qsize()) + tf.compat.v1.logging.warning('Bucket input queue is empty when calling next_batch. Bucket queue size: %i, Input queue size: %i', self._batch_queue.qsize(), self._example_queue.qsize()) if self._single_pass and self._finished_reading: - tf.logging.info("Finished reading dataset in single_pass mode.") + tf.compat.v1.logging.info("Finished reading dataset in single_pass mode.") return None batch = self._batch_queue.get() # get the next Batch @@ -324,11 +324,11 @@ def fill_example_queue(self): while True: try: - (article, abstract) = input_gen.next() # read the next example from file. article and abstract are both strings. + (article, abstract) = next(input_gen) # read the next example from file. article and abstract are both strings. except StopIteration: # if there are no more examples: - tf.logging.info("The example generator for this example queue filling thread has exhausted data.") + tf.compat.v1.logging.info("The example generator for this example queue filling thread has exhausted data.") if self._single_pass: - tf.logging.info("single_pass mode is on, so we've finished reading dataset. This thread is stopping.") + tf.compat.v1.logging.info("single_pass mode is on, so we've finished reading dataset. This thread is stopping.") self._finished_reading = True break else: @@ -348,13 +348,13 @@ def fill_batch_queue(self): if self._hps.mode != 'decode': # Get bucketing_cache_size-many batches of Examples into a list, then sort inputs = [] - for _ in xrange(self._hps.batch_size * self._bucketing_cache_size): + for _ in range(self._hps.batch_size * self._bucketing_cache_size): inputs.append(self._example_queue.get()) inputs = sorted(inputs, key=lambda inp: inp.enc_len) # sort by length of encoder sequence # Group the sorted Examples into batches, optionally shuffle the batches, and place in the batch queue. batches = [] - for i in xrange(0, len(inputs), self._hps.batch_size): + for i in range(0, len(inputs), self._hps.batch_size): batches.append(inputs[i:i + self._hps.batch_size]) if not self._single_pass: shuffle(batches) @@ -363,7 +363,7 @@ def fill_batch_queue(self): else: # beam search decode mode ex = self._example_queue.get() - b = [ex for _ in xrange(self._hps.batch_size)] + b = [ex for _ in range(self._hps.batch_size)] self._batch_queue.put(Batch(b, self._hps, self._vocab)) @@ -373,14 +373,14 @@ def watch_threads(self): time.sleep(60) for idx,t in enumerate(self._example_q_threads): if not t.is_alive(): # if the thread is dead - tf.logging.error('Found example queue thread dead. Restarting.') + tf.compat.v1.logging.error('Found example queue thread dead. Restarting.') new_t = Thread(target=self.fill_example_queue) self._example_q_threads[idx] = new_t new_t.daemon = True new_t.start() for idx,t in enumerate(self._batch_q_threads): if not t.is_alive(): # if the thread is dead - tf.logging.error('Found batch queue thread dead. Restarting.') + tf.compat.v1.logging.error('Found batch queue thread dead. Restarting.') new_t = Thread(target=self.fill_batch_queue) self._batch_q_threads[idx] = new_t new_t.daemon = True @@ -393,14 +393,14 @@ def text_generator(self, example_generator): Args: example_generator: a generator of tf.Examples from file. See data.example_generator""" while True: - e = example_generator.next() # e is a tf.Example + e = next(example_generator) # e is a tf.Example try: - article_text = e.features.feature['article'].bytes_list.value[0] # the article text was saved under the key 'article' in the data files - abstract_text = e.features.feature['abstract'].bytes_list.value[0] # the abstract text was saved under the key 'abstract' in the data files + article_text = e.features.feature['article'].bytes_list.value[0].decode('utf-8') # the article text was saved under the key 'article' in the data files + abstract_text = e.features.feature['abstract'].bytes_list.value[0].decode('utf-8') # the abstract text was saved under the key 'abstract' in the data files except ValueError: - tf.logging.error('Failed to get article or abstract from example') + tf.compat.v1.logging.error('Failed to get article or abstract from example') continue if len(article_text)==0: # See https://github.com/abisee/pointer-generator/issues/1 - tf.logging.warning('Found an example with empty article text. Skipping it.') + tf.compat.v1.logging.warning('Found an example with empty article text. Skipping it.') else: yield (article_text, abstract_text) diff --git a/beam_search.py b/beam_search.py old mode 100755 new mode 100644 index ff3e328..7ec1955 --- a/beam_search.py +++ b/beam_search.py @@ -20,7 +20,7 @@ import numpy as np import data -FLAGS = tf.app.flags.FLAGS +FLAGS = tf.compat.v1.app.flags.FLAGS class Hypothesis(object): """Class to represent a hypothesis during beam search. Holds all the information needed for the hypothesis.""" @@ -102,13 +102,13 @@ def run_beam_search(sess, model, vocab, batch): attn_dists=[], p_gens=[], coverage=np.zeros([batch.enc_batch.shape[1]]) # zero vector of length attention_length - ) for _ in xrange(FLAGS.beam_size)] + ) for _ in range(FLAGS.beam_size)] results = [] # this will contain finished hypotheses (those that have emitted the [STOP] token) steps = 0 while steps < FLAGS.max_dec_steps and len(results) < FLAGS.beam_size: latest_tokens = [h.latest_token for h in hyps] # latest token produced by each hypothesis - latest_tokens = [t if t in xrange(vocab.size()) else vocab.word2id(data.UNKNOWN_TOKEN) for t in latest_tokens] # change any in-article temporary OOV ids to [UNK] id, so that we can lookup word embeddings + latest_tokens = [t if t in range(vocab.size()) else vocab.word2id(data.UNKNOWN_TOKEN) for t in latest_tokens] # change any in-article temporary OOV ids to [UNK] id, so that we can lookup word embeddings states = [h.state for h in hyps] # list of current decoder states of the hypotheses prev_coverage = [h.coverage for h in hyps] # list of coverage vectors (or None) @@ -123,9 +123,9 @@ def run_beam_search(sess, model, vocab, batch): # Extend each hypothesis and collect them all in all_hyps all_hyps = [] num_orig_hyps = 1 if steps == 0 else len(hyps) # On the first step, we only had one original hypothesis (the initial hypothesis). On subsequent steps, all original hypotheses are distinct. - for i in xrange(num_orig_hyps): + for i in range(num_orig_hyps): h, new_state, attn_dist, p_gen, new_coverage_i = hyps[i], new_states[i], attn_dists[i], p_gens[i], new_coverage[i] # take the ith hypothesis and new decoder state info - for j in xrange(FLAGS.beam_size * 2): # for each of the top 2*beam_size hyps: + for j in range(FLAGS.beam_size * 2): # for each of the top 2*beam_size hyps: # Extend the ith hypothesis with the jth option new_hyp = h.extend(token=topk_ids[i, j], log_prob=topk_log_probs[i, j], diff --git a/data.py b/data.py old mode 100755 new mode 100644 index 37799a7..c7d3f5a --- a/data.py +++ b/data.py @@ -22,7 +22,8 @@ import csv from tensorflow.core.example import example_pb2 import gensim -from gensim import corpora, similarities, models +from gensim import corpora +import re # and are used in the data files to segment the abstracts into sentences. They don't receive vocab ids. SENTENCE_START = '' @@ -60,7 +61,7 @@ def __init__(self, vocab_file, max_size): for line in vocab_f: pieces = line.split() if len(pieces) != 2: - print 'Warning: incorrectly formatted line in vocabulary file: %s\n' % line + print('Warning: incorrectly formatted line in vocabulary file: %s\n' % line) continue w = pieces[0] if w in [SENTENCE_START, SENTENCE_END, UNKNOWN_TOKEN, PAD_TOKEN, START_DECODING, STOP_DECODING]: @@ -71,10 +72,10 @@ def __init__(self, vocab_file, max_size): self._id_to_word[self._count] = w self._count += 1 if max_size != 0 and self._count >= max_size: - print "max_size of vocab was specified as %i; we now have %i words. Stopping reading." % (max_size, self._count) + print("max_size of vocab was specified as %i; we now have %i words. Stopping reading." % (max_size, self._count)) break - print "Finished constructing vocabulary of %i total words. Last word added: %s" % (self._count, self._id_to_word[self._count-1]) + print("Finished constructing vocabulary of %i total words. Last word added: %s" % (self._count, self._id_to_word[self._count-1])) """The topic model is initialized here""" self.tm = TopicModel() @@ -104,26 +105,30 @@ def write_metadata(self, fpath): Args: fpath: place to write the metadata file """ - print "Writing word embedding metadata file to %s..." % (fpath) + print("Writing word embedding metadata file to %s..." % (fpath)) with open(fpath, "w") as f: fieldnames = ['word'] writer = csv.DictWriter(f, delimiter="\t", fieldnames=fieldnames) - for i in xrange(self.size()): + for i in range(self.size()): writer.writerow({"word": self._id_to_word[i]}) class TopicModel (object): """Reads a pretrained LDA model trained using the Gensim library""" - def __init__(self, modelAdd = '/projects/pointer_generator_networks/pointer-generator-topic-master/lda.model', dictAdd = '/projects/pointer_generator_networks/pointer-generator-topic-master/dictionary.dic', topicsFileAdd = ''): + def __init__(self, modelAdd = 'lda/lda.model', dictAdd = 'lda/dictionary.dic', topicsFileAdd = ''): self.topicModel, self.topicModelDictionary = self._loadPretrainedTM (modelAdd, dictAdd) self.topics_dictionary = self._createTopicsDict(1000)# A dictionary containing all topics + + # LDA needs to be retrained for this version of gensim, omitting errors by manually setting minimum_phi_value and per_word_topics + self.topicModel.per_word_topics = False + self.topicModel.minimum_phi_value = 0.01 def _loadPretrainedTM (self, modelAdd, dictAdd): lda = gensim.models.ldamodel.LdaModel.load(modelAdd, mmap = 'r') - print "Loaded the LDA model." + print("Loaded the LDA model.") dictionary = corpora.Dictionary.load(dictAdd, mmap = 'r') - print "Loaded dictionary." + print("Loaded dictionary.") return lda, dictionary def _doc2topics (self, doc): @@ -132,12 +137,12 @@ def _doc2topics (self, doc): return perDocTopics def _turnTopicOff (self, topicDict): - for k in topicDict.keys(): + for k in list(topicDict.keys()): topicDict[k] = 0.0 return topicDict def _createTopicsDict (self, numWordsPerTopic): # put all topics in a dictionary. A topic could be accessed by its number. Function returns a dictionary contating all topics. - print "Creating a topic dictionary" + print("Creating a topic dictionary") topics_dictionary = {} tm = self.topicModel.show_topics(num_topics=150, num_words = numWordsPerTopic, formatted=False) for t in tm: @@ -149,7 +154,7 @@ def _createTopicsDict (self, numWordsPerTopic): # put all topics in a dictionary #if t[0]==3 or t[0]==17: # temp = self._turnTopicOff(temp) topics_dictionary[t[0]]=temp - print "Finished building a dictionary of all topics" + print("Finished building a dictionary of all topics") return topics_dictionary @@ -159,9 +164,9 @@ def _doc2FinalWordVector (self, doc, topicProportions = True): #Get the topics of the given doc topics = self._doc2topics(doc) print (topics) - print (self.topics_dictionary.keys()) + print((list(self.topics_dictionary.keys()))) for tup in topics: - for item in self.topics_dictionary[tup[0]].iteritems(): + for item in self.topics_dictionary[tup[0]].items(): if item[0] in docTopics: if topicProportions: docTopics[item[0]]+=item[1]*tup[1] @@ -249,7 +254,7 @@ def example_generator(data_path, single_pass): example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0] yield example_pb2.Example.FromString(example_str) if single_pass: - print "example_generator completed reading all datafiles. No more data." + print("example_generator completed reading all datafiles. No more data.") break @@ -339,16 +344,7 @@ def abstract2sents(abstract): Returns: sents: List of sentence strings (no tags)""" - cur = 0 - sents = [] - while True: - try: - start_p = abstract.index(SENTENCE_START, cur) - end_p = abstract.index(SENTENCE_END, start_p + 1) - cur = end_p + len(SENTENCE_END) - sents.append(abstract[start_p+len(SENTENCE_START):end_p]) - except ValueError as e: # no more sentences - return sents + return re.findall(' (.+?) <\/s>', abstract) def show_art_oovs(article, vocab): diff --git a/decode.py b/decode.py old mode 100755 new mode 100644 index 90b5aec..5f16581 --- a/decode.py +++ b/decode.py @@ -27,7 +27,7 @@ import logging import numpy as np -FLAGS = tf.app.flags.FLAGS +FLAGS = tf.compat.v1.app.flags.FLAGS SECS_UNTIL_NEW_CKPT = 60 # max number of seconds before loading new checkpoint @@ -47,18 +47,22 @@ def __init__(self, model, batcher, vocab): self._model.build_graph() self._batcher = batcher self._vocab = vocab - self._saver = tf.train.Saver() # we use this to load checkpoints for decoding - self._sess = tf.Session(config=util.get_config()) + self._saver = tf.compat.v1.train.Saver() # we use this to load checkpoints for decoding + self._sess = tf.compat.v1.Session(config=util.get_config()) # Load an initial checkpoint to use for decoding ckpt_path = util.load_ckpt(self._saver, self._sess) if FLAGS.single_pass: - # Make a descriptive decode directory name - ckpt_name = "ckpt-" + ckpt_path.split('-')[-1] # this is something of the form "ckpt-123456" - self._decode_dir = os.path.join(FLAGS.log_root, get_decode_dir_name(ckpt_name)) - if os.path.exists(self._decode_dir): - raise Exception("single_pass decode directory %s should not already exist" % self._decode_dir) + if FLAGS.decode_dir != '': + # Overwrite automatic naming of directory + self._decode_dir = FLAGS.decode_dir + else: + # Make a descriptive decode directory name + ckpt_name = "ckpt-" + ckpt_path.split('-')[-1] # this is something of the form "ckpt-123456" + self._decode_dir = os.path.join(FLAGS.log_root, get_decode_dir_name(ckpt_name)) + if os.path.exists(self._decode_dir): + raise Exception("single_pass decode directory %s should not already exist" % self._decode_dir) else: # Generic decode dir name self._decode_dir = os.path.join(FLAGS.log_root, "decode") @@ -79,11 +83,14 @@ def decode(self): t0 = time.time() counter = 0 while True: + if FLAGS.single_input and counter == 1: + break + batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" - tf.logging.info("Decoder has finished reading dataset for single_pass.") - tf.logging.info("Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) + tf.compat.v1.logging.info("Decoder has finished reading dataset for single_pass.") + tf.compat.v1.logging.info("Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return @@ -120,7 +127,7 @@ def decode(self): # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1-t0 > SECS_UNTIL_NEW_CKPT: - tf.logging.info('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1-t0) + tf.compat.v1.logging.info('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1-t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time() @@ -159,7 +166,7 @@ def write_for_rouge(self, reference_sents, decoded_words, ex_index): for idx,sent in enumerate(decoded_sents): f.write(sent) if idx==len(decoded_sents)-1 else f.write(sent+"\n") - tf.logging.info("Wrote example %i to file" % ex_index) + tf.compat.v1.logging.info("Wrote example %i to file" % ex_index) def write_for_attnvis(self, article, abstract, decoded_words, attn_dists, p_gens): @@ -186,16 +193,16 @@ def write_for_attnvis(self, article, abstract, decoded_words, attn_dists, p_gens output_fname = os.path.join(self._decode_dir, 'attn_vis_data.json') with open(output_fname, 'w') as output_file: json.dump(to_write, output_file) - tf.logging.info('Wrote visualization data to %s', output_fname) + tf.compat.v1.logging.info('Wrote visualization data to %s', output_fname) def print_results(article, abstract, decoded_output): """Prints the article, the reference summmary and the decoded summary to screen""" - print "" - tf.logging.info('ARTICLE: %s', article) - tf.logging.info('REFERENCE SUMMARY: %s', abstract) - tf.logging.info('GENERATED SUMMARY: %s', decoded_output) - print "" + print("") + tf.compat.v1.logging.info('ARTICLE: %s', article) + tf.compat.v1.logging.info('REFERENCE SUMMARY: %s', abstract) + tf.compat.v1.logging.info('GENERATED SUMMARY: %s', decoded_output) + print("") def make_html_safe(s): @@ -234,9 +241,9 @@ def rouge_log(results_dict, dir_to_write): val_cb = results_dict[key_cb] val_ce = results_dict[key_ce] log_str += "%s: %.4f with confidence interval (%.4f, %.4f)\n" % (key, val, val_cb, val_ce) - tf.logging.info(log_str) # log to screen + tf.compat.v1.logging.info(log_str) # log to screen results_file = os.path.join(dir_to_write, "ROUGE_results.txt") - tf.logging.info("Writing final ROUGE results to %s...", results_file) + tf.compat.v1.logging.info("Writing final ROUGE results to %s...", results_file) with open(results_file, "w") as f: f.write(log_str) diff --git a/encode.py b/encode.py new file mode 100644 index 0000000..ba8338b --- /dev/null +++ b/encode.py @@ -0,0 +1,33 @@ +import struct +import tensorflow as tf +from tensorflow.core.example import example_pb2 +import stanza +import os +from pathlib import Path + +def tokenize_input(input_text): + """Maps input text to a tokenized version using Stanford CoreNLP Tokenizer""" + stanza.download('en') + + nlp = stanza.Pipeline('en', processors='tokenize') + doc = nlp(input_text) + + tokenized_text = '\n'.join([' '.join([token.text for token in sentence.tokens]) for sentence in doc.sentences]) + return tokenized_text + +def write_to_bin(input_text, out_file): + """Read the input text, tokenize it, and write it to an out_file""" + tokenized_input = tokenize_input(input_text) + + # Create directory for specific input file + Path(os.path.dirname(out_file)).mkdir(parents=True, exist_ok=True) + + with open(out_file, 'wb') as writer: + # Write to tf.Example + tf_example = example_pb2.Example() + tf_example.features.feature['article'].bytes_list.value.extend([tokenized_input.encode()]) + tf_example.features.feature['abstract'].bytes_list.value.extend([''.encode()]) + tf_example_str = tf_example.SerializeToString() + str_len = len(tf_example_str) + writer.write(struct.pack('q', str_len)) + writer.write(struct.pack('%ds' % str_len, tf_example_str)) \ No newline at end of file diff --git a/inspect_checkpoint.py b/inspect_checkpoint.py old mode 100755 new mode 100644 index 58d85a8..ccb56e5 --- a/inspect_checkpoint.py +++ b/inspect_checkpoint.py @@ -12,7 +12,7 @@ if len(sys.argv) != 2: raise Exception("Usage: python inspect_checkpoint.py \nNote: Do not include the .data .index or .meta part of the model checkpoint in file_name.") file_name = sys.argv[1] - reader = tf.train.NewCheckpointReader(file_name) + reader = tf.compat.v1.train.NewCheckpointReader(file_name) var_to_shape_map = reader.get_variable_to_shape_map() finite = [] @@ -29,17 +29,17 @@ else: some_infnan.append(key) - print "\nFINITE VARIABLES:" - for key in finite: print key + print("\nFINITE VARIABLES:") + for key in finite: print(key) - print "\nVARIABLES THAT ARE ALL INF/NAN:" - for key in all_infnan: print key + print("\nVARIABLES THAT ARE ALL INF/NAN:") + for key in all_infnan: print(key) - print "\nVARIABLES THAT CONTAIN SOME FINITE, SOME INF/NAN VALUES:" - for key in some_infnan: print key + print("\nVARIABLES THAT CONTAIN SOME FINITE, SOME INF/NAN VALUES:") + for key in some_infnan: print(key) - print "" + print("") if not all_infnan and not some_infnan: - print "CHECK PASSED: checkpoint contains no inf/NaN values" + print("CHECK PASSED: checkpoint contains no inf/NaN values") else: - print "CHECK FAILED: checkpoint contains some inf/NaN values" + print("CHECK FAILED: checkpoint contains some inf/NaN values") diff --git a/model.py b/model.py old mode 100755 new mode 100644 index 8a7b40d..a1438eb --- a/model.py +++ b/model.py @@ -15,15 +15,15 @@ # ============================================================================== """This file contains code to build and run the tensorflow graph for the sequence-to-sequence model""" -from __future__ import division + import os import time import numpy as np import tensorflow as tf from attention_decoder import attention_decoder -from tensorflow.contrib.tensorboard.plugins import projector +from tensorboard.plugins import projector -FLAGS = tf.app.flags.FLAGS +FLAGS = tf.compat.v1.app.flags.FLAGS class SummarizationModel(object): """A class to represent a sequence-to-sequence model for text summarization. Supports both baseline mode, pointer-generator mode, and coverage""" @@ -38,26 +38,26 @@ def _add_placeholders(self): hps = self._hps # encoder part - self._enc_batch = tf.placeholder(tf.int32, [hps.batch_size, None], name='enc_batch') - self._enc_lens = tf.placeholder(tf.int32, [hps.batch_size], name='enc_lens') - self._enc_padding_mask = tf.placeholder(tf.float32, [hps.batch_size, None], name='enc_padding_mask') + self._enc_batch = tf.compat.v1.placeholder(tf.int32, [hps.batch_size, None], name='enc_batch') + self._enc_lens = tf.compat.v1.placeholder(tf.int32, [hps.batch_size], name='enc_lens') + self._enc_padding_mask = tf.compat.v1.placeholder(tf.float32, [hps.batch_size, None], name='enc_padding_mask') # topic words part - self._enc_topicwords_batch = tf.placeholder (tf.int32, [hps.batch_size, None], name='enc_topicwords_batch') - self._enc_topicwords_probs_batch = tf.placeholder (tf.float32, [hps.batch_size, None], name='enc_topicwords_probs_batch') + self._enc_topicwords_batch = tf.compat.v1.placeholder (tf.int32, [hps.batch_size, None], name='enc_topicwords_batch') + self._enc_topicwords_probs_batch = tf.compat.v1.placeholder (tf.float32, [hps.batch_size, None], name='enc_topicwords_probs_batch') if FLAGS.pointer_gen: - self._enc_batch_extend_vocab = tf.placeholder(tf.int32, [hps.batch_size, None], name='enc_batch_extend_vocab') - self._max_art_oovs = tf.placeholder(tf.int32, [], name='max_art_oovs') + self._enc_batch_extend_vocab = tf.compat.v1.placeholder(tf.int32, [hps.batch_size, None], name='enc_batch_extend_vocab') + self._max_art_oovs = tf.compat.v1.placeholder(tf.int32, [], name='max_art_oovs') # decoder part - self._dec_batch = tf.placeholder(tf.int32, [hps.batch_size, hps.max_dec_steps], name='dec_batch') - self._target_batch = tf.placeholder(tf.int32, [hps.batch_size, hps.max_dec_steps], name='target_batch') - self._dec_padding_mask = tf.placeholder(tf.float32, [hps.batch_size, hps.max_dec_steps], name='dec_padding_mask') + self._dec_batch = tf.compat.v1.placeholder(tf.int32, [hps.batch_size, hps.max_dec_steps], name='dec_batch') + self._target_batch = tf.compat.v1.placeholder(tf.int32, [hps.batch_size, hps.max_dec_steps], name='target_batch') + self._dec_padding_mask = tf.compat.v1.placeholder(tf.float32, [hps.batch_size, hps.max_dec_steps], name='dec_padding_mask') if hps.mode=="decode" and hps.coverage: - self.prev_coverage = tf.placeholder(tf.float32, [hps.batch_size, None], name='prev_coverage') + self.prev_coverage = tf.compat.v1.placeholder(tf.float32, [hps.batch_size, None], name='prev_coverage') def _make_feed_dict(self, batch, just_enc=False): @@ -99,10 +99,10 @@ def _add_encoder(self, encoder_inputs, seq_len): fw_state, bw_state: Each are LSTMStateTuples of shape ([batch_size,hidden_dim],[batch_size,hidden_dim]) """ - with tf.variable_scope('encoder'): - cell_fw = tf.contrib.rnn.LSTMCell(self._hps.hidden_dim, initializer=self.rand_unif_init, state_is_tuple=True) - cell_bw = tf.contrib.rnn.LSTMCell(self._hps.hidden_dim, initializer=self.rand_unif_init, state_is_tuple=True) - (encoder_outputs, (fw_st, bw_st)) = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, encoder_inputs, dtype=tf.float32, sequence_length=seq_len, swap_memory=True) + with tf.compat.v1.variable_scope('encoder'): + cell_fw = tf.compat.v1.nn.rnn_cell.LSTMCell(self._hps.hidden_dim, initializer=self.rand_unif_init, state_is_tuple=True) + cell_bw = tf.compat.v1.nn.rnn_cell.LSTMCell(self._hps.hidden_dim, initializer=self.rand_unif_init, state_is_tuple=True) + (encoder_outputs, (fw_st, bw_st)) = tf.compat.v1.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, encoder_inputs, dtype=tf.float32, sequence_length=seq_len, swap_memory=True) encoder_outputs = tf.concat(axis=2, values=encoder_outputs) # concatenate the forwards and backwards states return encoder_outputs, fw_st, bw_st @@ -118,20 +118,20 @@ def _reduce_states(self, fw_st, bw_st): state: LSTMStateTuple with hidden_dim units. """ hidden_dim = self._hps.hidden_dim - with tf.variable_scope('reduce_final_st'): + with tf.compat.v1.variable_scope('reduce_final_st'): # Define weights and biases to reduce the cell and reduce the state - w_reduce_c = tf.get_variable('w_reduce_c', [hidden_dim * 2, hidden_dim], dtype=tf.float32, initializer=self.trunc_norm_init) - w_reduce_h = tf.get_variable('w_reduce_h', [hidden_dim * 2, hidden_dim], dtype=tf.float32, initializer=self.trunc_norm_init) - bias_reduce_c = tf.get_variable('bias_reduce_c', [hidden_dim], dtype=tf.float32, initializer=self.trunc_norm_init) - bias_reduce_h = tf.get_variable('bias_reduce_h', [hidden_dim], dtype=tf.float32, initializer=self.trunc_norm_init) + w_reduce_c = tf.compat.v1.get_variable('w_reduce_c', [hidden_dim * 2, hidden_dim], dtype=tf.float32, initializer=self.trunc_norm_init) + w_reduce_h = tf.compat.v1.get_variable('w_reduce_h', [hidden_dim * 2, hidden_dim], dtype=tf.float32, initializer=self.trunc_norm_init) + bias_reduce_c = tf.compat.v1.get_variable('bias_reduce_c', [hidden_dim], dtype=tf.float32, initializer=self.trunc_norm_init) + bias_reduce_h = tf.compat.v1.get_variable('bias_reduce_h', [hidden_dim], dtype=tf.float32, initializer=self.trunc_norm_init) # Apply linear layer old_c = tf.concat(axis=1, values=[fw_st.c, bw_st.c]) # Concatenation of fw and bw cell old_h = tf.concat(axis=1, values=[fw_st.h, bw_st.h]) # Concatenation of fw and bw state new_c = tf.nn.relu(tf.matmul(old_c, w_reduce_c) + bias_reduce_c) # Get new cell from old cell new_h = tf.nn.relu(tf.matmul(old_h, w_reduce_h) + bias_reduce_h) # Get new state from old state - return tf.contrib.rnn.LSTMStateTuple(new_c, new_h) # Return new cell and state + return tf.compat.v1.nn.rnn_cell.LSTMStateTuple(new_c, new_h) # Return new cell and state def _add_decoder(self, inputs): @@ -148,7 +148,7 @@ def _add_decoder(self, inputs): coverage: A tensor, the current coverage vector """ hps = self._hps - cell = tf.contrib.rnn.LSTMCell(hps.hidden_dim, state_is_tuple=True, initializer=self.rand_unif_init) + cell = tf.compat.v1.nn.rnn_cell.LSTMCell(hps.hidden_dim, state_is_tuple=True, initializer=self.rand_unif_init) prev_coverage = self.prev_coverage if hps.mode=="decode" and hps.coverage else None # In decode mode, we run attention_decoder one step at a time and so need to pass in the previous step's coverage vector each time @@ -168,7 +168,7 @@ def _calc_final_topicadded_dist(self, vocab_dists, attn_dists, topicwords_probs) final_dists: The final distributions. List length max_dec_steps of (batch_size, extended_vsize) arrays. """ - with tf.variable_scope('final_distribution'): + with tf.compat.v1.variable_scope('final_distribution'): #attn_dists = [(tf.nn.softmax(topicwords_probs) * attn_dist)/2 for attn_dist in attn_dists], this line is only used for turnning some topics off. Otherwise the following line should be used attn_dists = [(tf.nn.softmax(topicwords_probs) + attn_dist)/2 for attn_dist in attn_dists] #atten_dist changes over each decoder step while the topicwords_probs remain the same, This gives best results. Uncomment it! #attn_dists = [tf.nn.softmax(topicwords_probs) * attn_dist for attn_dist in attn_dists]# @@ -187,7 +187,7 @@ def _calc_final_topicadded_dist(self, vocab_dists, attn_dists, topicwords_probs) # This is fiddly; we use tf.scatter_nd to do the projection batch_nums = tf.range(0, limit=self._hps.batch_size) # shape (batch_size) batch_nums = tf.expand_dims(batch_nums, 1) # shape (batch_size, 1) - attn_len = tf.shape(self._enc_batch_extend_vocab)[1] # number of states we attend over + attn_len = tf.shape(input=self._enc_batch_extend_vocab)[1] # number of states we attend over batch_nums = tf.tile(batch_nums, [1, attn_len]) # shape (batch_size, attn_len) indices = tf.stack( (batch_nums, self._enc_batch_extend_vocab), axis=2) # shape (batch_size, enc_t, 2) shape = [self._hps.batch_size, extended_vsize] @@ -214,7 +214,7 @@ def _calc_final_dist(self, vocab_dists, attn_dists): Returns: final_dists: The final distributions. List length max_dec_steps of (batch_size, extended_vsize) arrays. """ - with tf.variable_scope('final_distribution'): + with tf.compat.v1.variable_scope('final_distribution'): # Multiply vocab dists by p_gen and attention dists by (1-p_gen) vocab_dists = [p_gen * dist for (p_gen,dist) in zip(self.p_gens, vocab_dists)] attn_dists = [(1-p_gen) * dist for (p_gen,dist) in zip(self.p_gens, attn_dists)] @@ -230,7 +230,7 @@ def _calc_final_dist(self, vocab_dists, attn_dists): # This is fiddly; we use tf.scatter_nd to do the projection batch_nums = tf.range(0, limit=self._hps.batch_size) # shape (batch_size) batch_nums = tf.expand_dims(batch_nums, 1) # shape (batch_size, 1) - attn_len = tf.shape(self._enc_batch_extend_vocab)[1] # number of states we attend over + attn_len = tf.shape(input=self._enc_batch_extend_vocab)[1] # number of states we attend over batch_nums = tf.tile(batch_nums, [1, attn_len]) # shape (batch_size, attn_len) indices = tf.stack( (batch_nums, self._enc_batch_extend_vocab), axis=2) # shape (batch_size, enc_t, 2) shape = [self._hps.batch_size, extended_vsize] @@ -251,7 +251,7 @@ def _add_emb_vis(self, embedding_var): train_dir = os.path.join(FLAGS.log_root, "train") vocab_metadata_path = os.path.join(train_dir, "vocab_metadata.tsv") self._vocab.write_metadata(vocab_metadata_path) # write metadata file - summary_writer = tf.summary.FileWriter(train_dir) + summary_writer = tf.compat.v1.summary.FileWriter(train_dir) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = embedding_var.name @@ -263,17 +263,17 @@ def _add_seq2seq(self): hps = self._hps vsize = self._vocab.size() # size of the vocabulary - with tf.variable_scope('seq2seq'): + with tf.compat.v1.variable_scope('seq2seq'): # Some initializers - self.rand_unif_init = tf.random_uniform_initializer(-hps.rand_unif_init_mag, hps.rand_unif_init_mag, seed=123) - self.trunc_norm_init = tf.truncated_normal_initializer(stddev=hps.trunc_norm_init_std) + self.rand_unif_init = tf.compat.v1.random_uniform_initializer(-hps.rand_unif_init_mag, hps.rand_unif_init_mag, seed=123) + self.trunc_norm_init = tf.compat.v1.truncated_normal_initializer(stddev=hps.trunc_norm_init_std) # Add embedding matrix (shared by the encoder and decoder inputs) - with tf.variable_scope('embedding'): - embedding = tf.get_variable('embedding', [vsize, hps.emb_dim], dtype=tf.float32, initializer=self.trunc_norm_init) + with tf.compat.v1.variable_scope('embedding'): + embedding = tf.compat.v1.get_variable('embedding', [vsize, hps.emb_dim], dtype=tf.float32, initializer=self.trunc_norm_init) if hps.mode=="train": self._add_emb_vis(embedding) # add to tensorboard - emb_enc_inputs = tf.nn.embedding_lookup(embedding, self._enc_batch) # tensor with shape (batch_size, max_enc_steps, emb_size) - emb_dec_inputs = [tf.nn.embedding_lookup(embedding, x) for x in tf.unstack(self._dec_batch, axis=1)] # list length max_dec_steps containing shape (batch_size, emb_size) + emb_enc_inputs = tf.nn.embedding_lookup(params=embedding, ids=self._enc_batch) # tensor with shape (batch_size, max_enc_steps, emb_size) + emb_dec_inputs = [tf.nn.embedding_lookup(params=embedding, ids=x) for x in tf.unstack(self._dec_batch, axis=1)] # list length max_dec_steps containing shape (batch_size, emb_size) # Add the encoder. enc_outputs, fw_st, bw_st = self._add_encoder(emb_enc_inputs, self._enc_lens) @@ -283,19 +283,19 @@ def _add_seq2seq(self): self._dec_in_state = self._reduce_states(fw_st, bw_st) # Add the decoder. - with tf.variable_scope('decoder'): + with tf.compat.v1.variable_scope('decoder'): decoder_outputs, self._dec_out_state, self.attn_dists, self.p_gens, self.coverage = self._add_decoder(emb_dec_inputs) # Add the output projection to obtain the vocabulary distribution - with tf.variable_scope('output_projection'): - w = tf.get_variable('w', [hps.hidden_dim, vsize], dtype=tf.float32, initializer=self.trunc_norm_init) - w_t = tf.transpose(w) - v = tf.get_variable('v', [vsize], dtype=tf.float32, initializer=self.trunc_norm_init) + with tf.compat.v1.variable_scope('output_projection'): + w = tf.compat.v1.get_variable('w', [hps.hidden_dim, vsize], dtype=tf.float32, initializer=self.trunc_norm_init) + w_t = tf.transpose(a=w) + v = tf.compat.v1.get_variable('v', [vsize], dtype=tf.float32, initializer=self.trunc_norm_init) vocab_scores = [] # vocab_scores is the vocabulary distribution before applying softmax. Each entry on the list corresponds to one decoder step for i,output in enumerate(decoder_outputs): if i > 0: - tf.get_variable_scope().reuse_variables() - vocab_scores.append(tf.nn.xw_plus_b(output, w, v)) # apply the linear layer + tf.compat.v1.get_variable_scope().reuse_variables() + vocab_scores.append(tf.compat.v1.nn.xw_plus_b(output, w, v)) # apply the linear layer vocab_dists = [tf.nn.softmax(s) for s in vocab_scores] # The vocabulary distributions. List length max_dec_steps of (batch_size, vsize) arrays. The words are in the order they appear in the vocabulary file. @@ -316,7 +316,7 @@ def _add_seq2seq(self): if hps.mode in ['train', 'eval']: # Calculate the loss - with tf.variable_scope('loss'): + with tf.compat.v1.variable_scope('loss'): if FLAGS.pointer_gen: # Calculate the loss per step # This is fiddly; we use tf.gather_nd to pick out the probabilities of the gold target words @@ -326,7 +326,7 @@ def _add_seq2seq(self): targets = self._target_batch[:,dec_step] # The indices of the target words. shape (batch_size) indices = tf.stack( (batch_nums, targets), axis=1) # shape (batch_size, 2) gold_probs = tf.gather_nd(dist, indices) # shape (batch_size). prob of correct words on this step - losses = -tf.log(gold_probs) + losses = -tf.math.log(gold_probs) loss_per_step.append(losses) # Apply dec_padding_mask and get loss @@ -335,15 +335,15 @@ def _add_seq2seq(self): else: # baseline model self._loss = tf.contrib.seq2seq.sequence_loss(tf.stack(vocab_scores, axis=1), self._target_batch, self._dec_padding_mask) # this applies softmax internally - tf.summary.scalar('loss', self._loss) + tf.compat.v1.summary.scalar('loss', self._loss) # Calculate coverage loss from the attention distributions if hps.coverage: - with tf.variable_scope('coverage_loss'): + with tf.compat.v1.variable_scope('coverage_loss'): self._coverage_loss = _coverage_loss(self.attn_dists, self._dec_padding_mask) - tf.summary.scalar('coverage_loss', self._coverage_loss) + tf.compat.v1.summary.scalar('coverage_loss', self._coverage_loss) self._total_loss = self._loss + hps.cov_loss_wt * self._coverage_loss - tf.summary.scalar('total_loss', self._total_loss) + tf.compat.v1.summary.scalar('total_loss', self._total_loss) if hps.mode == "decode": # We run decode beam search mode one decoder step at a time @@ -351,32 +351,32 @@ def _add_seq2seq(self): final_dists = final_dists[0] topk_probs, self._topk_ids = tf.nn.top_k(final_dists, hps.batch_size*2) # take the k largest probs. note batch_size=beam_size in decode mode - self._topk_log_probs = tf.log(topk_probs) + self._topk_log_probs = tf.math.log(topk_probs) def _add_train_op(self): """Sets self._train_op, the op to run for training.""" # Take gradients of the trainable variables w.r.t. the loss function to minimize loss_to_minimize = self._total_loss if self._hps.coverage else self._loss - tvars = tf.trainable_variables() - gradients = tf.gradients(loss_to_minimize, tvars, aggregation_method=tf.AggregationMethod.EXPERIMENTAL_TREE) + tvars = tf.compat.v1.trainable_variables() + gradients = tf.gradients(ys=loss_to_minimize, xs=tvars, aggregation_method=tf.AggregationMethod.EXPERIMENTAL_TREE) # Clip the gradients with tf.device("/gpu:0"): grads, global_norm = tf.clip_by_global_norm(gradients, self._hps.max_grad_norm) # Add a summary - tf.summary.scalar('global_norm', global_norm) + tf.compat.v1.summary.scalar('global_norm', global_norm) # Apply adagrad optimizer - optimizer = tf.train.AdagradOptimizer(self._hps.lr, initial_accumulator_value=self._hps.adagrad_init_acc) + optimizer = tf.compat.v1.train.AdagradOptimizer(self._hps.lr, initial_accumulator_value=self._hps.adagrad_init_acc) with tf.device("/gpu:0"): - self._train_op = optimizer.apply_gradients(zip(grads, tvars), global_step=self.global_step, name='train_step') + self._train_op = optimizer.apply_gradients(list(zip(grads, tvars)), global_step=self.global_step, name='train_step') def build_graph(self): """Add the placeholders, model, global step, train_op and summaries to the graph""" - tf.logging.info('Building graph...') + tf.compat.v1.logging.info('Building graph...') t0 = time.time() self._add_placeholders() with tf.device("/gpu:0"): @@ -386,9 +386,9 @@ def build_graph(self): self.global_step = tf.Variable(0, name='global_step', trainable=False) if self._hps.mode == 'train': self._add_train_op() - self._summaries = tf.summary.merge_all() + self._summaries = tf.compat.v1.summary.merge_all() t1 = time.time() - tf.logging.info('Time to build graph: %i seconds', t1 - t0) + tf.compat.v1.logging.info('Time to build graph: %i seconds', t1 - t0) def run_train_step(self, sess, batch): """Runs one training iteration. Returns a dictionary containing train op, summaries, loss, global_step and (optionally) coverage loss.""" @@ -431,7 +431,7 @@ def run_encoder(self, sess, batch): # dec_in_state is LSTMStateTuple shape ([batch_size,hidden_dim],[batch_size,hidden_dim]) # Given that the batch is a single example repeated, dec_in_state is identical across the batch so we just take the top row. - dec_in_state = tf.contrib.rnn.LSTMStateTuple(dec_in_state.c[0], dec_in_state.h[0]) + dec_in_state = tf.compat.v1.nn.rnn_cell.LSTMStateTuple(dec_in_state.c[0], dec_in_state.h[0]) return enc_states, dec_in_state @@ -464,7 +464,7 @@ def decode_onestep(self, sess, batch, latest_tokens, enc_states, dec_init_states hiddens = [np.expand_dims(state.h, axis=0) for state in dec_init_states] new_c = np.concatenate(cells, axis=0) # shape [batch_size,hidden_dim] new_h = np.concatenate(hiddens, axis=0) # shape [batch_size,hidden_dim] - new_dec_in_state = tf.contrib.rnn.LSTMStateTuple(new_c, new_h) + new_dec_in_state = tf.compat.v1.nn.rnn_cell.LSTMStateTuple(new_c, new_h) feed = { self._enc_states: enc_states, @@ -496,7 +496,7 @@ def decode_onestep(self, sess, batch, latest_tokens, enc_states, dec_init_states results = sess.run(to_return, feed_dict=feed) # run the decoder step # Convert results['states'] (a single LSTMStateTuple) into a list of LSTMStateTuple -- one for each hypothesis - new_states = [tf.contrib.rnn.LSTMStateTuple(results['states'].c[i, :], results['states'].h[i, :]) for i in xrange(beam_size)] + new_states = [tf.compat.v1.nn.rnn_cell.LSTMStateTuple(results['states'].c[i, :], results['states'].h[i, :]) for i in range(beam_size)] # Convert singleton list containing a tensor to a list of k arrays assert len(results['attn_dists'])==1 @@ -507,14 +507,14 @@ def decode_onestep(self, sess, batch, latest_tokens, enc_states, dec_init_states assert len(results['p_gens'])==1 p_gens = results['p_gens'][0].tolist() else: - p_gens = [None for _ in xrange(beam_size)] + p_gens = [None for _ in range(beam_size)] # Convert the coverage tensor to a list length k containing the coverage vector for each hypothesis if FLAGS.coverage: new_coverage = results['coverage'].tolist() assert len(new_coverage) == beam_size else: - new_coverage = [None for _ in xrange(beam_size)] + new_coverage = [None for _ in range(beam_size)] return results['ids'], results['probs'], new_states, attn_dists, p_gens, new_coverage @@ -530,10 +530,10 @@ def _mask_and_avg(values, padding_mask): a scalar """ - dec_lens = tf.reduce_sum(padding_mask, axis=1) # shape batch_size. float32 + dec_lens = tf.reduce_sum(input_tensor=padding_mask, axis=1) # shape batch_size. float32 values_per_step = [v * padding_mask[:,dec_step] for dec_step,v in enumerate(values)] values_per_ex = sum(values_per_step)/dec_lens # shape (batch_size); normalized value for each batch member - return tf.reduce_mean(values_per_ex) # overall average + return tf.reduce_mean(input_tensor=values_per_ex) # overall average def _coverage_loss(attn_dists, padding_mask): @@ -549,7 +549,7 @@ def _coverage_loss(attn_dists, padding_mask): coverage = tf.zeros_like(attn_dists[0]) # shape (batch_size, attn_length). Initial coverage is zero. covlosses = [] # Coverage loss per decoder timestep. Will be list length max_dec_steps containing shape (batch_size). for a in attn_dists: - covloss = tf.reduce_sum(tf.minimum(a, coverage), [1]) # calculate the coverage loss for this step + covloss = tf.reduce_sum(input_tensor=tf.minimum(a, coverage), axis=[1]) # calculate the coverage loss for this step covlosses.append(covloss) coverage += a # update the coverage vector coverage_loss = _mask_and_avg(covlosses, padding_mask) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..97213c5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +gensim +tensorflow +pyrouge +flask +uuid +stanza +pathlib \ No newline at end of file diff --git a/run_summarization.py b/run_summarization.py old mode 100755 new mode 100644 index cd1769d..44bc886 --- a/run_summarization.py +++ b/run_summarization.py @@ -16,7 +16,6 @@ """This is the top-level file to train, evaluate or test your summarization model""" -import sys import time import os import tensorflow as tf @@ -29,48 +28,53 @@ import util from tensorflow.python import debug as tf_debug -FLAGS = tf.app.flags.FLAGS +FLAGS = tf.compat.v1.app.flags.FLAGS + +# Disable eager execution - TF 1 to TF 2 migration +tf.compat.v1.disable_eager_execution() # Where to find data -tf.app.flags.DEFINE_string('data_path', '', 'Path expression to tf.Example datafiles. Can include wildcards to access multiple datafiles.') -tf.app.flags.DEFINE_string('vocab_path', '', 'Path expression to text vocabulary file.') +tf.compat.v1.app.flags.DEFINE_string('data_path', '', 'Path expression to tf.Example datafiles. Can include wildcards to access multiple datafiles.') +tf.compat.v1.app.flags.DEFINE_string('vocab_path', '', 'Path expression to text vocabulary file.') # Important settings -tf.app.flags.DEFINE_string('mode', 'train', 'must be one of train/eval/decode') -tf.app.flags.DEFINE_boolean('single_pass', False, 'For decode mode only. If True, run eval on the full dataset using a fixed checkpoint, i.e. take the current checkpoint, and use it to produce one summary for each example in the dataset, write the summaries to file and then get ROUGE scores for the whole dataset. If False (default), run concurrent decoding, i.e. repeatedly load latest checkpoint, use it to produce summaries for randomly-chosen examples and log the results to screen, indefinitely.') +tf.compat.v1.app.flags.DEFINE_string('mode', 'train', 'must be one of train/eval/decode') +tf.compat.v1.app.flags.DEFINE_boolean('single_pass', False, 'For decode mode only. If True, run eval on the full dataset using a fixed checkpoint, i.e. take the current checkpoint, and use it to produce one summary for each example in the dataset, write the summaries to file and then get ROUGE scores for the whole dataset. If False (default), run concurrent decoding, i.e. repeatedly load latest checkpoint, use it to produce summaries for randomly-chosen examples and log the results to screen, indefinitely.') +tf.compat.v1.app.flags.DEFINE_boolean('single_input', False, 'For decode mode only. If true, the process stops immediately after producing one summary. Can only be used when single_pass is also True.') # Where to save output -tf.app.flags.DEFINE_string('log_root', '', 'Root directory for all logging.') -tf.app.flags.DEFINE_string('exp_name', '', 'Name for experiment. Logs will be saved in a directory with this name, under log_root.') +tf.compat.v1.app.flags.DEFINE_string('log_root', '', 'Root directory for all logging.') +tf.compat.v1.app.flags.DEFINE_string('exp_name', '', 'Name for experiment. Logs will be saved in a directory with this name, under log_root.') +tf.compat.v1.app.flags.DEFINE_string('decode_dir', '', 'Name for the directory of a single input, single pass summarization. This overwrites default parameter-focused directory naming.') # Hyperparameters -tf.app.flags.DEFINE_integer('hidden_dim', 256, 'dimension of RNN hidden states') -tf.app.flags.DEFINE_integer('emb_dim', 128, 'dimension of word embeddings') -tf.app.flags.DEFINE_integer('batch_size', 16, 'minibatch size') -tf.app.flags.DEFINE_integer('max_enc_steps', 400, 'max timesteps of encoder (max source text tokens)') -tf.app.flags.DEFINE_integer('max_dec_steps', 100, 'max timesteps of decoder (max summary tokens)') -tf.app.flags.DEFINE_integer('beam_size', 5, 'beam size for beam search decoding.') -tf.app.flags.DEFINE_integer('min_dec_steps', 35, 'Minimum sequence length of generated summary. Applies only for beam search decoding mode') -tf.app.flags.DEFINE_integer('vocab_size', 50000, 'Size of vocabulary. These will be read from the vocabulary file in order. If the vocabulary file contains fewer words than this number, or if this number is set to 0, will take all words in the vocabulary file.') -tf.app.flags.DEFINE_float('lr', 0.15, 'learning rate') -tf.app.flags.DEFINE_float('adagrad_init_acc', 0.1, 'initial accumulator value for Adagrad') -tf.app.flags.DEFINE_float('rand_unif_init_mag', 0.02, 'magnitude for lstm cells random uniform inititalization') -tf.app.flags.DEFINE_float('trunc_norm_init_std', 1e-4, 'std of trunc norm init, used for initializing everything else') -tf.app.flags.DEFINE_float('max_grad_norm', 2.0, 'for gradient clipping') +tf.compat.v1.app.flags.DEFINE_integer('hidden_dim', 256, 'dimension of RNN hidden states') +tf.compat.v1.app.flags.DEFINE_integer('emb_dim', 128, 'dimension of word embeddings') +tf.compat.v1.app.flags.DEFINE_integer('batch_size', 16, 'minibatch size') +tf.compat.v1.app.flags.DEFINE_integer('max_enc_steps', 400, 'max timesteps of encoder (max source text tokens)') +tf.compat.v1.app.flags.DEFINE_integer('max_dec_steps', 100, 'max timesteps of decoder (max summary tokens)') +tf.compat.v1.app.flags.DEFINE_integer('beam_size', 5, 'beam size for beam search decoding.') +tf.compat.v1.app.flags.DEFINE_integer('min_dec_steps', 35, 'Minimum sequence length of generated summary. Applies only for beam search decoding mode') +tf.compat.v1.app.flags.DEFINE_integer('vocab_size', 50000, 'Size of vocabulary. These will be read from the vocabulary file in order. If the vocabulary file contains fewer words than this number, or if this number is set to 0, will take all words in the vocabulary file.') +tf.compat.v1.app.flags.DEFINE_float('lr', 0.15, 'learning rate') +tf.compat.v1.app.flags.DEFINE_float('adagrad_init_acc', 0.1, 'initial accumulator value for Adagrad') +tf.compat.v1.app.flags.DEFINE_float('rand_unif_init_mag', 0.02, 'magnitude for lstm cells random uniform inititalization') +tf.compat.v1.app.flags.DEFINE_float('trunc_norm_init_std', 1e-4, 'std of trunc norm init, used for initializing everything else') +tf.compat.v1.app.flags.DEFINE_float('max_grad_norm', 2.0, 'for gradient clipping') # Pointer-generator or baseline model -tf.app.flags.DEFINE_boolean('pointer_gen', True, 'If True, use pointer-generator model. If False, use baseline model.') +tf.compat.v1.app.flags.DEFINE_boolean('pointer_gen', True, 'If True, use pointer-generator model. If False, use baseline model.') # Coverage hyperparameters -tf.app.flags.DEFINE_boolean('coverage', False, 'Use coverage mechanism. Note, the experiments reported in the ACL paper train WITHOUT coverage until converged, and then train for a short phase WITH coverage afterwards. i.e. to reproduce the results in the ACL paper, turn this off for most of training then turn on for a short phase at the end.') -tf.app.flags.DEFINE_float('cov_loss_wt', 1.0, 'Weight of coverage loss (lambda in the paper). If zero, then no incentive to minimize coverage loss.') +tf.compat.v1.app.flags.DEFINE_boolean('coverage', False, 'Use coverage mechanism. Note, the experiments reported in the ACL paper train WITHOUT coverage until converged, and then train for a short phase WITH coverage afterwards. i.e. to reproduce the results in the ACL paper, turn this off for most of training then turn on for a short phase at the end.') +tf.compat.v1.app.flags.DEFINE_float('cov_loss_wt', 1.0, 'Weight of coverage loss (lambda in the paper). If zero, then no incentive to minimize coverage loss.') # Utility flags, for restoring and changing checkpoints -tf.app.flags.DEFINE_boolean('convert_to_coverage_model', False, 'Convert a non-coverage model to a coverage model. Turn this on and run in train mode. Your current training model will be copied to a new version (same name with _cov_init appended) that will be ready to run with coverage flag turned on, for the coverage training stage.') -tf.app.flags.DEFINE_boolean('restore_best_model', False, 'Restore the best model in the eval/ dir and save it in the train/ dir, ready to be used for further training. Useful for early stopping, or if your training checkpoint has become corrupted with e.g. NaN values.') +tf.compat.v1.app.flags.DEFINE_boolean('convert_to_coverage_model', False, 'Convert a non-coverage model to a coverage model. Turn this on and run in train mode. Your current training model will be copied to a new version (same name with _cov_init appended) that will be ready to run with coverage flag turned on, for the coverage training stage.') +tf.compat.v1.app.flags.DEFINE_boolean('restore_best_model', False, 'Restore the best model in the eval/ dir and save it in the train/ dir, ready to be used for further training. Useful for early stopping, or if your training checkpoint has become corrupted with e.g. NaN values.') # Debugging. See https://www.tensorflow.org/programmers_guide/debugger -tf.app.flags.DEFINE_boolean('debug', False, "Run in tensorflow's debug mode (watches for NaN/inf values)") +tf.compat.v1.app.flags.DEFINE_boolean('debug', False, "Run in tensorflow's debug mode (watches for NaN/inf values)") @@ -93,60 +97,60 @@ def calc_running_avg_loss(loss, running_avg_loss, summary_writer, step, decay=0. else: running_avg_loss = running_avg_loss * decay + (1 - decay) * loss running_avg_loss = min(running_avg_loss, 12) # clip - loss_sum = tf.Summary() + loss_sum = tf.compat.v1.Summary() tag_name = 'running_avg_loss/decay=%f' % (decay) loss_sum.value.add(tag=tag_name, simple_value=running_avg_loss) summary_writer.add_summary(loss_sum, step) - tf.logging.info('running_avg_loss: %f', running_avg_loss) + tf.compat.v1.logging.info('running_avg_loss: %f', running_avg_loss) return running_avg_loss def restore_best_model(): """Load bestmodel file from eval directory, add variables for adagrad, and save to train directory""" - tf.logging.info("Restoring bestmodel for training...") + tf.compat.v1.logging.info("Restoring bestmodel for training...") # Initialize all vars in the model - sess = tf.Session(config=util.get_config()) - print "Initializing all variables..." - sess.run(tf.initialize_all_variables()) + sess = tf.compat.v1.Session(config=util.get_config()) + print("Initializing all variables...") + sess.run(tf.compat.v1.initialize_all_variables()) # Restore the best model from eval dir - saver = tf.train.Saver([v for v in tf.all_variables() if "Adagrad" not in v.name]) - print "Restoring all non-adagrad variables from best model in eval dir..." + saver = tf.compat.v1.train.Saver([v for v in tf.compat.v1.all_variables() if "Adagrad" not in v.name]) + print("Restoring all non-adagrad variables from best model in eval dir...") curr_ckpt = util.load_ckpt(saver, sess, "eval") - print "Restored %s." % curr_ckpt + print("Restored %s." % curr_ckpt) # Save this model to train dir and quit new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model") new_fname = os.path.join(FLAGS.log_root, "train", new_model_name) - print "Saving model to %s..." % (new_fname) - new_saver = tf.train.Saver() # this saver saves all variables that now exist, including Adagrad variables + print("Saving model to %s..." % (new_fname)) + new_saver = tf.compat.v1.train.Saver() # this saver saves all variables that now exist, including Adagrad variables new_saver.save(sess, new_fname) - print "Saved." + print("Saved.") exit() def convert_to_coverage_model(): """Load non-coverage checkpoint, add initialized extra variables for coverage, and save as new checkpoint""" - tf.logging.info("converting non-coverage model to coverage model..") + tf.compat.v1.logging.info("converting non-coverage model to coverage model..") # initialize an entire coverage model from scratch - sess = tf.Session(config=util.get_config()) - print "initializing everything..." - sess.run(tf.global_variables_initializer()) + sess = tf.compat.v1.Session(config=util.get_config()) + print("initializing everything...") + sess.run(tf.compat.v1.global_variables_initializer()) # load all non-coverage weights from checkpoint - saver = tf.train.Saver([v for v in tf.global_variables() if "coverage" not in v.name and "Adagrad" not in v.name]) - print "restoring non-coverage variables..." + saver = tf.compat.v1.train.Saver([v for v in tf.compat.v1.global_variables() if "coverage" not in v.name and "Adagrad" not in v.name]) + print("restoring non-coverage variables...") curr_ckpt = util.load_ckpt(saver, sess) - print "restored." + print("restored.") # save this model and quit new_fname = curr_ckpt + '_cov_init' - print "saving model to %s..." % (new_fname) - new_saver = tf.train.Saver() # this one will save all variables that now exist + print("saving model to %s..." % (new_fname)) + new_saver = tf.compat.v1.train.Saver() # this one will save all variables that now exist new_saver.save(sess, new_fname) - print "saved." + print("saved.") exit() @@ -161,9 +165,9 @@ def setup_training(model, batcher): convert_to_coverage_model() if FLAGS.restore_best_model: restore_best_model() - saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time + saver = tf.compat.v1.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time - sv = tf.train.Supervisor(logdir=train_dir, + sv = tf.compat.v1.train.Supervisor(logdir=train_dir, is_chief=True, saver=saver, summary_op=None, @@ -171,19 +175,19 @@ def setup_training(model, batcher): save_model_secs=60, # checkpoint every 60 secs global_step=model.global_step) summary_writer = sv.summary_writer - tf.logging.info("Preparing or waiting for session...") + tf.compat.v1.logging.info("Preparing or waiting for session...") sess_context_manager = sv.prepare_or_wait_for_session(config=util.get_config()) - tf.logging.info("Created session.") + tf.compat.v1.logging.info("Created session.") try: run_training(model, batcher, sess_context_manager, sv, summary_writer) # this is an infinite loop until interrupted except KeyboardInterrupt: - tf.logging.info("Caught keyboard interrupt on worker. Stopping supervisor...") + tf.compat.v1.logging.info("Caught keyboard interrupt on worker. Stopping supervisor...") sv.stop() def run_training(model, batcher, sess_context_manager, sv, summary_writer): """Repeatedly runs training iterations, logging loss to screen and writing summaries""" - tf.logging.info("starting run_training") + tf.compat.v1.logging.info("starting run_training") with sess_context_manager as sess: if FLAGS.debug: # start the tensorflow debugger sess = tf_debug.LocalCLIDebugWrapperSession(sess) @@ -191,21 +195,21 @@ def run_training(model, batcher, sess_context_manager, sv, summary_writer): while True: # repeats until interrupted batch = batcher.next_batch() - tf.logging.info('running training step...') + tf.compat.v1.logging.info('running training step...') t0=time.time() results = model.run_train_step(sess, batch) t1=time.time() - tf.logging.info('seconds for training step: %.3f', t1-t0) + tf.compat.v1.logging.info('seconds for training step: %.3f', t1-t0) loss = results['loss'] - tf.logging.info('loss: %f', loss) # print the loss to screen + tf.compat.v1.logging.info('loss: %f', loss) # print the loss to screen if not np.isfinite(loss): raise Exception("Loss is not finite. Stopping.") if FLAGS.coverage: coverage_loss = results['coverage_loss'] - tf.logging.info("coverage_loss: %f", coverage_loss) # print the coverage loss to screen + tf.compat.v1.logging.info("coverage_loss: %f", coverage_loss) # print the coverage loss to screen # get the summaries and iteration number so we can write summaries to tensorboard summaries = results['summaries'] # we will write these summaries to tensorboard using summary_writer @@ -219,11 +223,11 @@ def run_training(model, batcher, sess_context_manager, sv, summary_writer): def run_eval(model, batcher, vocab): """Repeatedly runs eval iterations, logging to screen and writing summaries. Saves the model with the best loss seen so far.""" model.build_graph() # build the graph - saver = tf.train.Saver(max_to_keep=3) # we will keep 3 best checkpoints at a time - sess = tf.Session(config=util.get_config()) + saver = tf.compat.v1.train.Saver(max_to_keep=3) # we will keep 3 best checkpoints at a time + sess = tf.compat.v1.Session(config=util.get_config()) eval_dir = os.path.join(FLAGS.log_root, "eval") # make a subdir of the root dir for eval data bestmodel_save_path = os.path.join(eval_dir, 'bestmodel') # this is where checkpoints of best models are saved - summary_writer = tf.summary.FileWriter(eval_dir) + summary_writer = tf.compat.v1.summary.FileWriter(eval_dir) running_avg_loss = 0 # the eval job keeps a smoother, running average loss to tell it when to implement early stopping best_loss = None # will hold the best loss achieved so far @@ -235,14 +239,14 @@ def run_eval(model, batcher, vocab): t0=time.time() results = model.run_eval_step(sess, batch) t1=time.time() - tf.logging.info('seconds for batch: %.2f', t1-t0) + tf.compat.v1.logging.info('seconds for batch: %.2f', t1-t0) # print the loss and coverage loss to screen loss = results['loss'] - tf.logging.info('loss: %f', loss) + tf.compat.v1.logging.info('loss: %f', loss) if FLAGS.coverage: coverage_loss = results['coverage_loss'] - tf.logging.info("coverage_loss: %f", coverage_loss) + tf.compat.v1.logging.info("coverage_loss: %f", coverage_loss) # add summaries summaries = results['summaries'] @@ -255,7 +259,7 @@ def run_eval(model, batcher, vocab): # If running_avg_loss is best so far, save this checkpoint (early stopping). # These checkpoints will appear as bestmodel- in the eval dir if best_loss is None or running_avg_loss < best_loss: - tf.logging.info('Found new best model with %.3f running_avg_loss. Saving to %s', running_avg_loss, bestmodel_save_path) + tf.compat.v1.logging.info('Found new best model with %.3f running_avg_loss. Saving to %s', running_avg_loss, bestmodel_save_path) saver.save(sess, bestmodel_save_path, global_step=train_step, latest_filename='checkpoint_best') best_loss = running_avg_loss @@ -268,8 +272,8 @@ def main(unused_argv): if len(unused_argv) != 1: # prints a message if you've entered flags incorrectly raise Exception("Problem with flags: %s" % unused_argv) - tf.logging.set_verbosity(tf.logging.INFO) # choose what level of logging you want - tf.logging.info('Starting seq2seq_attention in %s mode...', (FLAGS.mode)) + tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) # choose what level of logging you want + tf.compat.v1.logging.info('Starting seq2seq_attention in %s mode...', (FLAGS.mode)) # Change log_root to FLAGS.log_root/FLAGS.exp_name and create the dir if necessary FLAGS.log_root = os.path.join(FLAGS.log_root, FLAGS.exp_name) @@ -294,17 +298,17 @@ def main(unused_argv): # Make a namedtuple hps, containing the values of the hyperparameters that the model needs hparam_list = ['mode', 'lr', 'adagrad_init_acc', 'rand_unif_init_mag', 'trunc_norm_init_std', 'max_grad_norm', 'hidden_dim', 'emb_dim', 'batch_size', 'max_dec_steps', 'max_enc_steps', 'coverage', 'cov_loss_wt', 'pointer_gen'] hps_dict = {} - for key,val in FLAGS.__flags.iteritems(): # for each flag + for key,val in FLAGS.flag_values_dict().items(): # for each flag if key in hparam_list: # if it's in the list hps_dict[key] = val # add it to the dict - hps = namedtuple("HParams", hps_dict.keys())(**hps_dict) + hps = namedtuple("HParams", list(hps_dict.keys()))(**hps_dict) # Create a batcher object that will create minibatches of data batcher = Batcher(FLAGS.data_path, vocab, hps, single_pass=FLAGS.single_pass) - tf.set_random_seed(111) # a seed value for randomness + tf.compat.v1.set_random_seed(111) # a seed value for randomness if hps.mode == 'train': - print "creating model..." + print("creating model...") model = SummarizationModel(hps, vocab) setup_training(model, batcher) elif hps.mode == 'eval': @@ -320,4 +324,4 @@ def main(unused_argv): raise ValueError("The 'mode' flag must be one of train/eval/decode") if __name__ == '__main__': - tf.app.run() + tf.compat.v1.app.run() diff --git a/util.py b/util.py old mode 100755 new mode 100644 index b4e8b1f..39eba92 --- a/util.py +++ b/util.py @@ -19,11 +19,11 @@ import tensorflow as tf import time import os -FLAGS = tf.app.flags.FLAGS +FLAGS = tf.compat.v1.app.flags.FLAGS def get_config(): """Returns config for tf.session""" - config = tf.ConfigProto(allow_soft_placement=True) + config = tf.compat.v1.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth=True return config @@ -34,9 +34,9 @@ def load_ckpt(saver, sess, ckpt_dir="train"): latest_filename = "checkpoint_best" if ckpt_dir=="eval" else None ckpt_dir = os.path.join(FLAGS.log_root, ckpt_dir) ckpt_state = tf.train.get_checkpoint_state(ckpt_dir, latest_filename=latest_filename) - tf.logging.info('Loading checkpoint %s', ckpt_state.model_checkpoint_path) + tf.compat.v1.logging.info('Loading checkpoint %s', ckpt_state.model_checkpoint_path) saver.restore(sess, ckpt_state.model_checkpoint_path) return ckpt_state.model_checkpoint_path except: - tf.logging.info("Failed to load checkpoint from %s. Sleeping for %i secs...", ckpt_dir, 10) + tf.compat.v1.logging.info("Failed to load checkpoint from %s. Sleeping for %i secs...", ckpt_dir, 10) time.sleep(10)