Здравствуйте, продолжение статьи про методы создания эмбедингов предложений. В этом гайде мало слов и много кода, готово для Ctrl+с, Ctrl+v, улучшений и дальнейших тестов.
Часть1 обязательна для ознакомления
4. BERT
from deeppavlov.core.common.file import read_json
from deeppavlov import build_model, configs
from deeppavlov.models.embedders.elmo_embedder import ELMoEmbedder
# ссылка для скачивания моделей http://docs.deeppavlov.ai/en/master/features/pretrained_vectors.html4.1 rubert_cased_L-12_H-768_A-12_pt
class RU_BERT_CLASS:
def __init__(self, name):
bert_config = read_json(configs.embedder.bert_embedder)
bert_config['metadata']['variables']['BERT_PATH'] = os.path.join('./.', name)
self.m = build_model(bert_config)
def vectorizer(self, sentences):
return [sentence.split() for sentence in sentences]
def predict(self, tokens):
_, _, _, _, sent_max_embs, sent_mean_embs, _ = self.m(tokens)
return sent_mean_embs
bert = RU_BERT_CLASS('rubert_cased_L-12_H-768_A-12_pt')
get_similarity_values = similarity_values_wrapper(bert.predict, bert.vectorizer, distance_function=cosine_distances)
evaluate(get_similarity_values, 'rubert')'rubert: 2895.7'
4.2 ru_conversational_cased_L-12_H-768_A-12_pt
bert = RU_BERT_CLASS('ru_conversational_cased_L-12_H-768_A-12_pt')
get_similarity_values = similarity_values_wrapper(bert.predict, bert.vectorizer, distance_function=cosine_distances)
evaluate(get_similarity_values, 'ru_conversational')'ru_conversational: 3559.1'
4.3 sentence_ru_cased_L-12_H-768_A-12_pt
bert = RU_BERT_CLASS('sentence_ru_cased_L-12_H-768_A-12_pt')
get_similarity_values = similarity_values_wrapper(bert.predict, bert.vectorizer, distance_function=cosine_distances)
evaluate(get_similarity_values, 'sentence_ru')'sentence_ru: 2660.2'
4.4 elmo_ru-news_wmt11-16_1.5M_steps
class ELMO_CLASS(RU_BERT_CLASS):
def __init__(self, name):
self.m = ELMoEmbedder(f"http://files.deeppavlov.ai/deeppavlov_data/{name}")
def predict(self, tokens):
return self.m(tokens)elmo = ELMO_CLASS('elmo_ru-news_wmt11-16_1.5M_steps.tar.gz')
get_similarity_values = similarity_values_wrapper(elmo.predict, elmo.vectorizer, distance_function=cosine_distances)
evaluate(get_similarity_values, 'elmo_ru-news')'elmo_ru-news: 4631.3'
4.5 elmo_ru-wiki_600k_steps
elmo = ELMO_CLASS('elmo_ru-wiki_600k_steps.tar.gz')
get_similarity_values = similarity_values_wrapper(elmo.predict, elmo.vectorizer, distance_function=cosine_distances)
evaluate(get_similarity_values, 'elmo_ru-wiki')'elmo_ru-wiki: 4507.6'
4.6 elmo_ru-twitter_2013-01_2018-04_600k_steps
elmo = ELMO_CLASS('elmo_ru-twitter_2013-01_2018-04_600k_steps.tar.gz')
get_similarity_values = similarity_values_wrapper(elmo.predict, elmo.vectorizer, distance_function=cosine_distances)
evaluate(get_similarity_values, 'elmo_ru-twitter')'elmo_ru-twitter: 2962.2'
plot_results()
5. Автоэнкодеры
Автоэнкодеры созданы для сжатия многомерного ветора до одномерного и, теоретически, должны идеально подойти для создания эмбедингов предложения.
5.1 Автоэнкодер embedings -> embedings
def models_builder(data_generator):
def cosine_loss(y_true, y_pred):
return K.mean(cosine_similarity(y_true, y_pred, axis=-1))
complexity = 300
inp = Input(shape=(data_generator.max_len, data_generator.embedding_size))
X = inp
X = Bidirectional(LSTM(complexity, return_sequences=True))(X)
X = Bidirectional(LSTM(int(complexity/10), return_sequences=True))(X)
X = Flatten()(X)
X = Dense(complexity, activation='elu')(X)
X = Dense(complexity, activation='elu')(X)
X = Dense(complexity, activation='linear', name='embeding_output')(X)
X = Dense(complexity, activation='elu')(X)
X = Dense(data_generator.max_len*complexity, activation='elu')(X)
X = Reshape((data_generator.max_len, complexity))(X)
X = Bidirectional(LSTM(complexity, return_sequences=True))(X)
X = Bidirectional(LSTM(complexity, return_sequences=True))(X)
X = Dense(data_generator.embedding_size, activation='elu')(X)
autoencoder = Model(inputs=inp, outputs=X)
autoencoder.compile(loss=cosine_loss, optimizer='adam')
autoencoder.summary()
embedder = Model(inputs=inp, outputs=autoencoder.get_layer('embeding_output').output)
return autoencoder, embedder
data_generator = EmbedingsDataGenerator(use_fasttext=False)
autoencoder, embedder = models_builder(data_generator)
get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize, distance_function=cosine_distances)new_result = -10e5
for i in tqdm(range(1000)):
if i%3==0:
previous_result = new_result
new_result = evaluate(get_similarity_values, 'автоэнкодер embedings -> embedings')
new_result = parse_result(new_result)
print(i, new_result)
if new_result < previous_result and i > 20:
break
for x, y in data_generator:
autoencoder.train_on_batch(x, x)0 1770.2
3 212.6
6 138.8
9 84.8
12 78.1
15 106.4
18 112.7
21 79.7
5.2 Автоэнкодер embedings -> indexes
def models_builder(data_generator):
complexity = 300
inp = Input(shape=(data_generator.max_len, data_generator.embedding_size))
X = inp
X = Bidirectional(LSTM(complexity, return_sequences=True))(X)
X = Bidirectional(LSTM(int(complexity/10), return_sequences=True))(X)
X = Flatten()(X)
X = Dense(complexity, activation='elu')(X)
X = Dense(complexity, activation='elu')(X)
X = Dense(complexity, activation='linear', name='embeding_output')(X)
X = Dense(complexity, activation='elu')(X)
X = Dense(data_generator.max_len*complexity, activation='elu')(X)
X = Reshape((data_generator.max_len, complexity))(X)
X = Bidirectional(LSTM(complexity, return_sequences=True))(X)
X = Bidirectional(LSTM(complexity, return_sequences=True))(X)
X = Dense(len(data_generator.token2index), activation='softmax')(X)
autoencoder = Model(inputs=inp, outputs=X)
autoencoder.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
autoencoder.summary()
embedder = Model(inputs=inp, outputs=autoencoder.get_layer('embeding_output').output)
return autoencoder, embedder
data_generator = IndexesDataGenerator()
autoencoder, embedder = models_builder(data_generator)
get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)new_result = -10e5
for i in tqdm(range(1000)):
if i%3==0:
previous_result = new_result
new_result = evaluate(get_similarity_values, 'автоэнкодер embedings -> indexes')
new_result = parse_result(new_result)
print(i, new_result)
if new_result < previous_result and i > 20:
break
for x_e, x_i, y_i in data_generator:
autoencoder.train_on_batch(x_e, x_i)0 1352.9
3 43.6
6 41.7
9 8.1
12 -5.6
15 43.1
18 36.1
21 -3.7
5.3 Автоэнкодер архитектура LSTM -> LSTM
def models_builder(data_generator):
def cosine_loss(y_true, y_pred):
return K.mean(cosine_similarity(y_true, y_pred, axis=-1))
complexity = 300
inp = Input(shape=(data_generator.max_len, data_generator.embedding_size))
X = inp
X, state_h, state_c = LSTM(complexity, return_state=True)(X)
X = Concatenate()([state_h, state_c])
X = Dense(complexity, activation='linear', name='embeding_output')(X)
state_c = Dense(complexity, activation='linear')(X)
state_h = Dense(complexity, activation='linear')(X)
inp_zeros = Input(shape=(data_generator.max_len, data_generator.embedding_size))
X = LSTM(complexity, return_sequences=True)(inp_zeros, [state_c, state_h])
X = Dense(data_generator.embedding_size, activation='linear')(X)
autoencoder = Model(inputs=[inp, inp_zeros], outputs=X)
autoencoder.compile(loss=cosine_loss, optimizer='adam')
autoencoder.summary()
embedder = Model(inputs=inp, outputs=autoencoder.get_layer('embeding_output').output)
return autoencoder, embedder
data_generator = EmbedingsDataGenerator(use_fasttext=False)
autoencoder, embedder = models_builder(data_generator)
get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)zeros = np.zeros((data_generator.batch_size, data_generator.max_len, data_generator.embedding_size))
new_result = -10e5
for i in tqdm(range(1000)):
if i%3==0:
previous_result = new_result
new_result = evaluate(get_similarity_values, 'автоэнкодер embedings -> indexes')
new_result = parse_result(new_result)
print(i, new_result)
if new_result < previous_result and i > 20:
break
for x, y in data_generator:
autoencoder.train_on_batch([x, zeros], x)0 1903.6
3 1299.3
6 313.5
9 445.3
12 454.9
15 447.7
18 454.5
21 448.1
5.4 Автоэнкодер архитектура LSTM -> LSTM -> indexes
def models_builder(data_generator):
complexity = 300
inp = Input(shape=(data_generator.max_len, data_generator.embedding_size))
X = inp
X, state_h, state_c = LSTM(complexity, return_state=True)(X)
X = Concatenate()([state_h, state_c])
X = Dense(complexity, activation='linear', name='embeding_output')(X)
state_c = Dense(complexity, activation='linear')(X)
state_h = Dense(complexity, activation='linear')(X)
inp_zeros = Input(shape=(data_generator.max_len, data_generator.embedding_size))
X = LSTM(complexity, return_sequences=True)(inp_zeros, [state_c, state_h])
X = Dense(len(data_generator.token2index), activation='softmax')(X)
autoencoder = Model(inputs=[inp, inp_zeros], outputs=X)
autoencoder.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
autoencoder.summary()
embedder = Model(inputs=inp, outputs=autoencoder.get_layer('embeding_output').output)
return autoencoder, embedder
data_generator = IndexesDataGenerator()
autoencoder, embedder = models_builder(data_generator)
get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)zeros = np.zeros((data_generator.batch_size, data_generator.max_len, data_generator.embedding_size))
new_result = -10e5
for i in tqdm(range(1000)):
if i%3==0:
previous_result = new_result
new_result = evaluate(get_similarity_values, 'автоэнкодер архитектура LSTM -> LSTM -> indexes')
new_result = parse_result(new_result)
print(i, new_result)
if new_result < previous_result and i > 20:
break
for x_e, x_i, y_i in data_generator:
autoencoder.train_on_batch([x_e, zeros], x_i)0 1903.6
3 1483.3
6 1249.3
9 566.3
12 789.2
15 702.3
18 480.5
21 552.3
24 533.0
Методы с учителем
6. Эмбединги на Transfer Learning
TEXTS_CORPUS_WITH_LABEL = [(sentence, topic) for topic in texts_for_training for sentence in texts_for_training[topic]]
class BowDataGenerator(EmbedingsDataGenerator):
def __init__(self, texts_topics=TEXTS_CORPUS_WITH_LABEL, batch_size=128, batches_per_epoch=100):
self.texts_topics = texts_topics
self.topic2index = {topic: index for index, topic in enumerate({topic for text, topic in self.texts_topics})}
self.batch_size = batch_size
self.batches_per_epoch = batches_per_epoch
self.count_vectorizer = CountVectorizer().fit([text_topic[0] for text_topic in self.texts_topics])
counts = Counter([text_topic[1] for text_topic in self.texts_topics])
self.class_weight = {self.topic2index[intent_id]:1/counts[intent_id] for intent_id in counts}
def vectorize(self, sentences):
return self.count_vectorizer.transform(sentences).toarray()
def __iter__(self):
for _ in tqdm(range(self.batches_per_epoch), leave=False):
X_batch = []
y_batch = []
finished_batch = False
while not finished_batch:
text, topic = random.choice(self.texts_topics)
X_batch.append(text)
y_batch.append(self.topic2index[topic])
if len(X_batch) >= self.batch_size:
X_batch = self.count_vectorizer.transform(X_batch).toarray()
y_batch = to_categorical(y_batch, num_classes=len(self.topic2index))
yield np.array(X_batch), np.array(y_batch)
finished_batch = True
data_generator = BowDataGenerator()6.1 Эмбединги на основе BOW
def models_builder(data_generator):
complexity = 500
inp = Input(shape=(len(data_generator.count_vectorizer.get_feature_names()),))
X = inp
X = Dense(complexity)(X)
X = Activation('elu')(X)
X = Dense(complexity)(X)
X = Activation('elu')(X)
X = Dense(complexity, name='embeding_output')(X)
X = Activation('elu')(X)
X = Dense(len(data_generator.topic2index), activation='softmax')(X)
model = Model(inputs=inp, outputs=X)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
model.summary()
embedder = Model(inputs=inp, outputs=model.get_layer('embeding_output').output)
return model, embedder
data_generator = BowDataGenerator()
model, embedder = models_builder(data_generator)
get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)new_result = -10e5
for i in tqdm(range(1000)):
if i%3==0:
previous_result = new_result
new_result = evaluate(get_similarity_values, 'ембединг на BOW')
new_result = parse_result(new_result)
print(i, new_result)
if new_result < previous_result and i > 20:
break
for x, y in data_generator:
model.train_on_batch(x, y, class_weight=data_generator.class_weight)0 601.4
3 1175.4
6 1187.0
9 1175.9
12 1097.9
15 1083.4
18 1083.8
21 1060.5
6.2 Эмбединг на LSTM + MaxPooling (InferSent)
Сыылки на стать:
Arxiv с теорией
Объяснено по-человечески
class LabelsDataGenerator(EmbedingsDataGenerator):
def __init__(self, texts_topics=TEXTS_CORPUS_WITH_LABEL, target_len=20, batch_size=128, batches_per_epoch=100, use_word2vec=True, use_fasttext=True):
self.texts_topics = texts_topics
self.topic2index = {topic: index for index, topic in enumerate({topic for text, topic in self.texts_topics})}
self.target_len = target_len
self.batch_size = batch_size
self.batches_per_epoch = batches_per_epoch
self.use_word2vec = use_word2vec
self.use_fasttext = use_fasttext
self.embedding_size = len(vectorize('token', use_word2vec=self.use_word2vec, use_fasttext=self.use_fasttext))
counts = Counter([text_topic[1] for text_topic in self.texts_topics])
self.class_weight = {self.topic2index[intent_id]:1/counts[intent_id] for intent_id in counts}
def vectorize(self, sentences):
vectorized = []
for text in sentences:
tokens = str(text).split()
x_vec = []
for token in tokens:
token_vec = vectorize(token, use_word2vec=self.use_word2vec, use_fasttext=self.use_fasttext)
x_vec.append(token_vec)
vectorized.append(x_vec)
vectorized = pad_sequences(vectorized, maxlen=self.target_len)
return vectorized
def __iter__(self):
for _ in tqdm(range(self.batches_per_epoch), leave=False):
X_batch = []
y_batch = []
finished_batch = False
while not finished_batch:
text, topic = random.choice(self.texts_topics)
tokens = text.split()
x_vec = []
for token in tokens:
token_vec = vectorize(token, use_word2vec=self.use_word2vec, use_fasttext=self.use_fasttext)
if len(x_vec) >= self.target_len:
X_batch.append(x_vec)
y_batch.append(self.topic2index[topic])
if len(X_batch) >= self.batch_size:
break
x_vec.append(token_vec)
else:
X_batch.append(x_vec)
y_batch.append(self.topic2index[topic])
if len(X_batch) >= self.batch_size:
X_batch = pad_sequences(X_batch, maxlen=self.target_len)
y_batch = to_categorical(y_batch, num_classes=len(self.topic2index))
yield np.array(X_batch), np.array(y_batch)
finished_batch = Truedef models_builder(data_generator):
complexity = 768
inp = Input(shape=(data_generator.target_len, data_generator.embedding_size))
X = inp
X = Bidirectional(LSTM(complexity, return_sequences=True))(X)
X = Permute((2,1))(X)
X = MaxPooling1D(pool_size=600)(X)
X = Flatten()(X)
X = Dense(complexity)(X)
X = Activation('elu')(X)
X = Dense(complexity, name='embeding_output')(X)
X = Activation('sigmoid')(X)
X = Dense(len(data_generator.topic2index), activation='softmax')(X)
model = Model(inputs=inp, outputs=X)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
model.summary()
embedder = Model(inputs=inp, outputs=model.get_layer('embeding_output').output)
return model, embedder
data_generator = LabelsDataGenerator()
model, embedder = models_builder(data_generator)
get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)new_result = -10e5
for i in tqdm(range(1000)):
if i%3==0:
previous_result = new_result
new_result = evaluate(get_similarity_values, 'эмбединг на LSTM + MaxPooling')
new_result = parse_result(new_result)
print(i, new_result)
if new_result < previous_result and i > 20:
break
for x, y in data_generator:
model.train_on_batch(x, y, class_weight=data_generator.class_weight)0 87.0
3 152.1
6 110.5
9 146.7
12 166.2
15 79.8
18 47.2
21 84.0
24 144.8
27 83.8
6.3 Эмбединг на LSTM + Conv1D + AveragePooling
def models_builder(data_generator):
complexity = 600
inp = Input(shape=(data_generator.target_len, data_generator.embedding_size))
X_R = inp
X_R = Bidirectional(LSTM(complexity, return_sequences=True))(X_R)
X_R = Bidirectional(LSTM(complexity, return_sequences=True))(X_R)
X_C = inp
X_C = Conv1D(complexity, 3, strides=1, padding='same')(X_C)
X_C = Conv1D(complexity, 3, strides=1, padding='same')(X_C)
X = Concatenate()([X_R, X_C])
X = AveragePooling1D(pool_size=2)(X)
X = Conv1D(complexity, 3, strides=1, padding='same')(X)
X = AveragePooling1D(pool_size=2)(X)
X = Conv1D(complexity, 3, strides=1, padding='same')(X)
X = AveragePooling1D(pool_size=2)(X)
X = Flatten()(X)
X = Dense(complexity)(X)
X = Activation('sigmoid')(X)
X = Dense(complexity, name = 'embeding_output')(X)
X = Activation('elu')(X)
X = Dense(len(data_generator.topic2index), activation='softmax')(X)
model = Model(inputs=inp, outputs=X)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
model.summary()
embedder = Model(inputs=inp, outputs=model.get_layer('embeding_output').output)
return model, embedder
data_generator = LabelsDataGenerator()
model, embedder = models_builder(data_generator)
get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)0 353.8
3 -147.8
6 7.6
9 5.5
12 -133.6
15 -133.6
18 9.0
21 9.0
24 -133.6
6.4 Эмбединг на LSTM + Inception + Attention
def models_builder(data_generator):
rate = 0.20
complexity = 500
def inception_convolutional_layer(X, complexity, rate=0.2, regularizer=0):
X_7 = Conv1D(int(complexity/7), kernel_size=7, strides=1, padding='same')(X)
X_6 = Conv1D(int(complexity/6), kernel_size=6, strides=1, padding='same')(X)
X_5 = Conv1D(int(complexity/5), kernel_size=5, strides=1, padding='same')(X)
X_4 = Conv1D(int(complexity/4), kernel_size=4, strides=1, padding='same')(X)
X_3 = Conv1D(int(complexity/3), kernel_size=3, strides=1, padding='same')(X)
X_2 = Conv1D(int(complexity/2), kernel_size=2, strides=1, padding='same')(X)
X_1 = Conv1D(int(complexity/1), kernel_size=1, strides=1, padding='same')(X)
X = Concatenate()([X_7, X_6, X_5, X_4, X_3, X_2, X_1])
X = Activation('elu')(X)
X = BatchNormalization()(X)
X = Dropout(rate)(X)
return X
def bi_LSTM(X, complexity, rate=0.2, regularizer=0):
X = Bidirectional(LSTM(int(complexity/2), return_sequences=True))(X)
X = BatchNormalization()(X)
X = Dropout(rate)(X)
return X
def dense_layer(X, complexity, activation='elu', rate=0.2, regularizer=0, name=None):
X = Dense(int(complexity), name=name)(X)
X = Activation(activation)(X)
X = BatchNormalization()(X)
X = Dropout(rate)(X)
return X
inp = Input(shape=(data_generator.target_len, data_generator.embedding_size))
X = inp
X = inception_convolutional_layer(X, complexity)
X = inception_convolutional_layer(X, complexity)
X = inception_convolutional_layer(X, complexity)
X = MaxPooling1D(pool_size=2)(X)
X = inception_convolutional_layer(X, complexity)
X = MaxPooling1D(pool_size=2)(X)
X = inception_convolutional_layer(X, complexity)
X = MaxPooling1D(pool_size=2)(X)
R = inp
R = bi_LSTM(R, complexity)
R = bi_LSTM(R, complexity/2)
attention_probs = Dense(int(complexity/2), activation='sigmoid', name='attention_probs')(R)
R = multiply([R, attention_probs], name='attention_mul')
R = Dropout(rate)(R)
R = MaxPooling1D(pool_size=2)(R)
R = inception_convolutional_layer(R, complexity)
R = MaxPooling1D(pool_size=2)(R)
R = inception_convolutional_layer(R, complexity)
R = MaxPooling1D(pool_size=2)(R)
X = Concatenate(axis=-1)([X, R])
X = Flatten()(X)
X = BatchNormalization()(X)
X = Dropout(rate)(X)
X = dense_layer(X, complexity)
X = dense_layer(X, complexity, activation='sigmoid')
X = dense_layer(X, complexity, name='embeding_output')
X = Dense(len(data_generator.topic2index), activation='softmax')(X)
model = Model(inputs=inp, outputs=X)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
model.summary()
embedder = Model(inputs=inp, outputs=model.get_layer('embeding_output').output)
return model, embedder
data_generator = LabelsDataGenerator()
model, embedder = models_builder(data_generator)
get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)new_result = -10e5
for i in tqdm(range(1000)):
if i%3==0:
previous_result = new_result
new_result = evaluate(get_similarity_values, 'эмбединг на LSTM + Inception + Attention')
new_result = parse_result(new_result)
print(i, new_result)
if new_result < previous_result and i > 20:
break
for x, y in data_generator:
model.train_on_batch(x, y, class_weight=data_generator.class_weight)0 275.0
3 126.8
6 173.9
9 155.5
12 168.4
15 287.2
18 382.8
21 303.4
plot_results()
7 Triplet loss
Обучение будет происходит на том, что векторы из одной темы должны распологаться ближе друг к другу, а из разных тем, дальше. Тем самым предложения, иемющие похожий смысл будут стоять ближ друг к другу, а разный, будут отстоять друг от друга.
Подробнее про Triplet loss вот тут
7.1 Triplet loss на BOW
class TripletDataGeneratorIndexes(BowDataGenerator):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.database = {}
for text, topic in self.texts_topics:
if topic not in self.database:
self.database[topic] = []
self.database[topic].append(text)
# почистим все интенты с <5 сообщениями
sh_database = {}
for topic in self.database:
if len(self.database[topic]) > 5:
sh_database[topic] = self.database[topic]
self.database = sh_database
self.all_topics = [topic for topic in self.database]
def __iter__(self):
for _ in tqdm(range(self.batches_per_epoch), leave=False):
anchor = []
positive = []
negative = []
for _ in range(self.batch_size):
anchor_topic = random.choice(self.all_topics)
anchor_index = np.random.randint(len(self.database[anchor_topic]))
positive_index = np.random.randint(len(self.database[anchor_topic]))
while positive_index == anchor_index:
positive_index = np.random.randint(len(self.database[anchor_topic]))
negative_topic = random.choice(self.all_topics)
while negative_topic == anchor_topic:
negative_topic = random.choice(self.all_topics)
negative_index = np.random.randint(len(self.database[negative_topic]))
anchor.append(self.database[anchor_topic][anchor_index])
positive.append(self.database[anchor_topic][positive_index])
negative.append(self.database[negative_topic][negative_index])
yield self.vectorize(anchor), self.vectorize(positive), self.vectorize(negative)def models_builder(data_generator):
sentence_embeding_size = 100
def lossless_triplet_loss(y_true, y_pred, N=sentence_embeding_size, beta=100, epsilon=1e-8):
"""
Implementation of the triplet loss function
Arguments:
y_true -- true labels, required when you define a loss in Keras, you don't need it in this function.
y_pred -- python list containing three objects:
anchor -- the encodings for the anchor data
positive -- the encodings for the positive data (similar to anchor)
negative -- the encodings for the negative data (different from anchor)
N -- The number of dimension
beta -- The scaling factor, N is recommended
epsilon -- The Epsilon value to prevent ln(0)
Returns:
loss -- real number, value of the loss
"""
anchor = tf.convert_to_tensor(y_pred[:,0:N])
positive = tf.convert_to_tensor(y_pred[:,N:N*2])
negative = tf.convert_to_tensor(y_pred[:,N*2:N*3])
# distance between the anchor and the positive
pos_dist = tf.reduce_sum(tf.square(tf.subtract(anchor,positive)),1)
# distance between the anchor and the negative
neg_dist = tf.reduce_sum(tf.square(tf.subtract(anchor,negative)),1)
#Non Linear Values
pos_dist = -tf.math.log(-tf.math.divide((pos_dist),beta)+1+epsilon)
neg_dist = -tf.math.log(-tf.math.divide((N-neg_dist),beta)+1+epsilon)
# compute loss
loss = neg_dist + pos_dist
return loss
def basic_sentence_vectorizer():
inp = Input(shape=(len(data_generator.count_vectorizer.get_feature_names()),))
X = inp
X = Dense(complexity)(X)
X = Activation('elu')(X)
X = Dense(complexity)(X)
X = Activation('elu')(X)
X = Dense(complexity, name='embeding_output')(X)
X = Activation('elu')(X)
X = Dense(complexity)(X)
vectorizer = Model(inputs=inp, outputs=X)
return vectorizer
complexity = 300
inp_anchor = Input(shape=(len(data_generator.count_vectorizer.get_feature_names()),))
inp_positive = Input(shape=(len(data_generator.count_vectorizer.get_feature_names()),))
inp_negative = Input(shape=(len(data_generator.count_vectorizer.get_feature_names()),))
embedder = basic_sentence_vectorizer()
anchor = embedder(inp_anchor)
positive = embedder(inp_positive)
negative = embedder(inp_negative)
output = Concatenate(axis=1)([anchor, positive, negative])
model = Model(inputs=[inp_anchor, inp_positive, inp_negative], outputs=output)
model.compile(optimizer='adagrad', loss=lossless_triplet_loss)
model.summary()
return model, embedder
data_generator = TripletDataGeneratorIndexes(batch_size=128, batches_per_epoch=10000)
model, embedder = models_builder(data_generator)
get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)zeros = np.zeros((data_generator.batch_size, 1, 1))
new_result = -10e5
for i in tqdm(range(1000)):
if i%3==0:
previous_result = new_result
new_result = evaluate(get_similarity_values, 'triplet loss indexes')
new_result = parse_result(new_result)
print(i, new_result)
if new_result < previous_result and i > 20:
break
for a, p, n in data_generator:
model.train_on_batch([a, p, n], zeros)0 724.1
3 -143.5
6 11.7
9 36.2
12 -123.5
15 150.1
18 -51.9
21 5.0
24 -43.5
7.2 Triplet loss на embedings
class TripletDataGeneratorEmbedings(TripletDataGeneratorIndexes):
def __init__(self, *args, **kwargs):
super().__init__()
self.target_len = kwargs['target_len']
self.embedding_size = len(vectorize('any_token'))
self.use_word2vec = True
self.use_fasttext = True
self.batches_per_epoch = kwargs['batches_per_epoch']
def vectorize(self, sentences):
return LabelsDataGenerator.vectorize(self, sentences)def models_builder(data_generator):
sentence_embeding_size = 300
def lossless_triplet_loss(y_true, y_pred, N=sentence_embeding_size, beta=100, epsilon=1e-8):
"""
Implementation of the triplet loss function
Arguments:
y_true -- true labels, required when you define a loss in Keras, you don't need it in this function.
y_pred -- python list containing three objects:
anchor -- the encodings for the anchor data
positive -- the encodings for the positive data (similar to anchor)
negative -- the encodings for the negative data (different from anchor)
N -- The number of dimension
beta -- The scaling factor, N is recommended
epsilon -- The Epsilon value to prevent ln(0)
Returns:
loss -- real number, value of the loss
"""
anchor = tf.convert_to_tensor(y_pred[:,0:N])
positive = tf.convert_to_tensor(y_pred[:,N:N*2])
negative = tf.convert_to_tensor(y_pred[:,N*2:N*3])
# distance between the anchor and the positive
pos_dist = tf.math.reduce_sum(tf.math.square(tf.math.subtract(anchor,positive)),1)
# distance between the anchor and the negative
neg_dist = tf.math.reduce_sum(tf.math.square(tf.math.subtract(anchor,negative)),1)
#Non Linear Values
pos_dist = -tf.math.log(-tf.math.divide((pos_dist),beta)+1+epsilon)
neg_dist = -tf.math.log(-tf.math.divide((N-neg_dist),beta)+1+epsilon)
# compute loss
loss = neg_dist + pos_dist
return loss
def inception_convolutional_layer(X, complexity, rate=0.2, regularizer=0):
X_7 = Conv1D(int(complexity/7), kernel_size=7, strides=1, padding='same')(X)
X_6 = Conv1D(int(complexity/6), kernel_size=6, strides=1, padding='same')(X)
X_5 = Conv1D(int(complexity/5), kernel_size=5, strides=1, padding='same')(X)
X_4 = Conv1D(int(complexity/4), kernel_size=4, strides=1, padding='same')(X)
X_3 = Conv1D(int(complexity/3), kernel_size=3, strides=1, padding='same')(X)
X_2 = Conv1D(int(complexity/2), kernel_size=2, strides=1, padding='same')(X)
X_1 = Conv1D(int(complexity/1), kernel_size=1, strides=1, padding='same')(X)
X = Concatenate()([X_7, X_6, X_5, X_4, X_3, X_2, X_1])
X = Activation('elu')(X)
X = BatchNormalization()(X)
X = Dropout(rate)(X)
return X
def bi_LSTM(X, complexity, rate=0.2, regularizer=0):
X = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(int(complexity/2), return_sequences=True))(X)
X = tf.keras.layers.BatchNormalization()(X)
X = tf.keras.layers.Dropout(rate)(X)
return X
def dense_layer(X, complexity, rate=0.2, regularizer=0):
X = tf.keras.layers.Dense(int(complexity))(X)
X = tf.keras.layers.Activation('elu')(X)
X = tf.keras.layers.BatchNormalization()(X)
X = tf.keras.layers.Dropout(rate)(X)
return X
def basic_sentence_vectorizer():
rate = 0.20
complexity = 300
inp = Input(shape = (data_generator.target_len, data_generator.embedding_size))
X = inp
X = inception_convolutional_layer(X, complexity)
X = inception_convolutional_layer(X, complexity)
X = inception_convolutional_layer(X, complexity)
X = tf.keras.layers.MaxPooling1D(pool_size=2)(X)
X = inception_convolutional_layer(X, complexity)
X = tf.keras.layers.MaxPooling1D(pool_size=2)(X)
X = inception_convolutional_layer(X, complexity)
X = tf.keras.layers.MaxPooling1D(pool_size=2)(X)
R = inp
R = bi_LSTM(R, complexity)
R = bi_LSTM(R, complexity/2)
attention_probs = tf.keras.layers.Dense(int(complexity/2), activation='sigmoid', name='attention_probs')(R)
R = multiply([R, attention_probs], name='attention_mul')
R = Dropout(rate)(R)
R = MaxPooling1D(pool_size=2)(R)
R = inception_convolutional_layer(R, complexity)
R = MaxPooling1D(pool_size=2)(R)
R = inception_convolutional_layer(R, complexity)
R = MaxPooling1D(pool_size=2)(R)
X = Concatenate(axis=-1)([X, R])
X = Flatten()(X)
X = BatchNormalization()(X)
X = Dropout(rate)(X)
X = dense_layer(X, complexity)
X = dense_layer(X, complexity)
X = dense_layer(X, complexity)
X = Dense(sentence_embeding_size, activation='sigmoid')(X)
vectorizer = Model(inputs=inp, outputs=X)
return vectorizer
inp_anchor = Input(shape = (data_generator.target_len, data_generator.embedding_size))
inp_positive = Input(shape = (data_generator.target_len, data_generator.embedding_size))
inp_negative = Input(shape = (data_generator.target_len, data_generator.embedding_size))
embedder = basic_sentence_vectorizer()
anchor = embedder(inp_anchor)
positive = embedder(inp_positive)
negative = embedder(inp_negative)
output = Concatenate(axis=1)([anchor, positive, negative])
model = Model(inputs=[inp_anchor, inp_positive, inp_negative], outputs=output)
model.compile(optimizer='adagrad', loss=lossless_triplet_loss)
model.summary()
return model, embedder
data_generator = TripletDataGeneratorEmbedings(target_len=20, batch_size=32, batches_per_epoch=10000)
model, embedder = models_builder(data_generator)
get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)zeros = np.zeros((data_generator.batch_size, 1, 1))
new_result = -10e5
for i in tqdm(range(1000)):
if i%3==0:
previous_result = new_result
new_result = evaluate(get_similarity_values, 'triplet loss embeding')
new_result = parse_result(new_result)
print(i, new_result)
if new_result < previous_result and i>20:
break
for a, p, n in data_generator:
model.train_on_batch([a, p, n], zeros)0 283.9
3 334.2
6 218.1
9 219.6
12 262.8
15 282.4
18 289.7
21 274.9
plot_results()
Итоги
Можно было предсказать, что победителями будут модели ELMO т.к. они были созданы для векторизации предложений. Их можно смело использовать, когда вам нужно быстро извлечь фичи из текста.
Лично меня приятно удивил BOW и среднее по эмбедингам. Даже без учёта порядка слов, они смогли поставить предложения из одной темы рядом.
Был разочарован автоэнкодерами. Сразу после инициализации результат лучше, чем после обучения. Не могу сказать в чём проблема, скорее всего автоэнкодер не может сжать всё предложение правильно и начинает предсказывать нули. Если у вас будут идеи по улучшению, то жду в комментариях.
Мой личный фаворит Triplet loss на embedings тоже не дал выдающегося результата. Думаю, что он раскроет свой потенциал на моделях в 100 раз больше по размеру и с обучением в течении нескольких месяцев.
Два метода: BOW с леммами без стоп слов и среднее с весами tf-idf хоть и не дают выдающихся средних результатов, но для некоторых предложений дают очень и очень хороший результат. Поэтому, для этих методов, всё должно зависеть от данных.
Вероятно, что со временем будет и Часть 3, если наберу достаточное количество идей.