Answer the question
In order to leave comments, you need to log in
What's wrong with the generator?
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import skipgrams
from keras.models import Sequential
from keras.layers import Embedding, Dense, Flatten
from keras.utils import np_utils
sentences = [line.strip() for line in open('alice_in_wonderland.txt') if line != '\n']
tokenizer = Tokenizer()
tokenizer.fit_on_texts(sentences)
corpus = tokenizer.texts_to_sequences(sentences)
V = len(tokenizer.index_word)+1
window_size = 1
model = Sequential()
model.add(Embedding(V, 64, input_length = 1))
model.add(Dense(V, activation = 'softmax'))
model.compile(loss= 'categorical_crossentropy', optimizer = 'rmsprop')
model.summary()
def generate_data(corpus, window_size, V):
for sequence in corpus:
idknow = skipgrams(sequence, V, window_size, negative_samples=0., shuffle= True)
X, y = zip(*idknow[0])
yield X, y
for epoch in range(10):
loss = 0.
for x, y in generate_data(corpus, window_size, V):
print(x)
print(y)
loss += model.train_on_batch(x, y)
print(loss)
Answer the question
In order to leave comments, you need to log in
Didn't find what you were looking for?
Ask your questionAsk a Question
731 491 924 answers to any question