-
Notifications
You must be signed in to change notification settings - Fork 0
/
FoV_seq2seq_mu_var.py
337 lines (278 loc) · 15.6 KB
/
FoV_seq2seq_mu_var.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
'''
- An encoder LSTM turns input sequences to 2 state vectors
(we keep the last LSTM state and discard the outputs).
- A decoder LSTM is trained to turn the target sequences into
the same sequence but offset by one timestep in the future,
a training process called "teacher forcing" in this context.
Is uses as initial state the state vectors from the encoder.
Effectively, the decoder learns to generate `targets[t+1...]`
given `targets[...t]`, conditioned on the input sequence.
- In inference mode, when we want to decode unknown input sequences, we:
- Encode the input sequence into state vectors
- Start with a target sequence of size 1
(just the start-of-sequence character)
- Feed the state vectors and 1-char target sequence
to the decoder to produce predictions for the next character
- Sample the next character using these predictions
(we simply use argmax).
- Append the sampled character to the target sequence
- Repeat until we generate the end-of-sequence character or we
hit the character limit.
'''
from __future__ import print_function
from keras.models import Model
from keras.layers import Input, LSTM, Dense
from keras.callbacks import ModelCheckpoint,ReduceLROnPlateau,EarlyStopping
import sys,glob,io,random
if '/scratch/wz1219/FoV' not in sys.path:
sys.path.insert(0, '/scratch/wz1219/FoV')
from mycode.dataLayer import DataLayer
import mycode.cost as costfunc
# from mycode.provide_hidden_state import multilayer_perceptron_hidden_state_series,multilayer_perceptron_hidden_state,dynamicRNN_hidden_state
from mycode.config import cfg
from mycode.dataIO import clip_xyz
import _pickle as pickle
import numpy as np
import matplotlib.pyplot as plt
import pdb
batch_size = 64 # Batch size for training.
epochs = 200 # Number of epochs to train for.
latent_dim = 64 # Latent dimensionality of the encoding space.
fps = 30
num_encoder_tokens = 3*fps
num_decoder_tokens = 6
max_encoder_seq_length = cfg.running_length
max_decoder_seq_length = cfg.predict_step
num_user = 48
## data preparation
if cfg.use_xyz:
#all_video_data = pickle.load(open('/scratch/wz1219/FoV/data/exp_1_code.p','rb'))
all_video_data = pickle.load(open('/scratch/wz1219/FoV/data/exp_1_xyz.p','rb'))
data_dim = 3
#all_video_data = clip_xyz(all_video_data)
datadb = all_video_data.copy()
def reshape2second_stacks(per_video_db,collapse_user=False):
"""reshape from N* *90 into M*max_encoder_seq_length*90"""
# split into chunks of max_encoder_seq_length seconds
per_video_db = np.array(np.split(per_video_db,per_video_db.shape[1]/max_encoder_seq_length,axis=1))
# decoder target
per_video_db_future = per_video_db[1:,:,:,:]
last_location = per_video_db[:-1,:,-1,:][:,:,np.newaxis,:]
# decoder input
per_video_db_future_input = np.concatenate((last_location,per_video_db_future[:,:,:-1,:]),axis=2)
if collapse_user:
#mix user and time
#reshape, collapse user dimension
per_video_db = per_video_db[:-1,:,:,:]
per_video_db = np.reshape(per_video_db,(-1,max_encoder_seq_length,num_encoder_tokens))
per_video_db_future = np.reshape(per_video_db_future,(-1,max_encoder_seq_length,num_encoder_tokens))
per_video_db_future_input = np.reshape(per_video_db_future_input,(-1,max_encoder_seq_length,num_encoder_tokens))
else:
#keep the user dimension in dim 0
#userful for other users' data
per_video_db = per_video_db[:-1,:,:,:]
per_video_db = per_video_db.transpose((1,0,2,3))
per_video_db_future = per_video_db_future.transpose((1,0,2,3))
per_video_db_future_input = per_video_db_future_input.transpose((1,0,2,3))
return per_video_db,per_video_db_future,per_video_db_future_input
def get_data(datadb,pick_user=False):
if not pick_user:
### concat all 9 videos and all users
# don't distinguish users or videos during training
_video_db = np.zeros((1,max_encoder_seq_length,num_encoder_tokens))
_video_db_future = np.zeros((1,max_encoder_seq_length,num_encoder_tokens))
_video_db_future_input = np.zeros((1,max_encoder_seq_length,num_encoder_tokens))
for _video_ind in datadb.keys():
per_video_db = np.stack((datadb[_video_ind]['x'],datadb[_video_ind]['y'],datadb[_video_ind]['z']),axis=-1)
#per_video_db = np.stack((datadb[_video_ind]['one_hot_code_row'],datadb[_video_ind]['one_hot_code_col']),axis = -1)
per_video_db = per_video_db[:,per_video_db.shape[1]-per_video_db.shape[1]/fps/max_encoder_seq_length*fps*max_encoder_seq_length:,:] #cut head to make dividable by seconds
#one_hot_matrix = np.zeros((per_video_db.shape[0],per_video_db.shape[1],72))
#for i in range(per_video_db.shape[0]):
# for j in range(per_video_db.shape[1]):
# temp = np.zeros((6,12))
# temp[int(per_video_db[i,j,0]),int(per_video_db[i,j,1])] = 1
# one_hot_matrix[i,j,:] = temp.flatten()
# print(int(per_video_db[i,j,0]),int(per_video_db[i,j,1]))
#per_video_db = one_hot_matrix
per_video_db = np.reshape(per_video_db,(per_video_db.shape[0],per_video_db.shape[1]/fps,num_encoder_tokens))
per_video_db, per_video_db_future, per_video_db_future_input = reshape2second_stacks(per_video_db,collapse_user=True)
_video_db = np.concatenate((_video_db, per_video_db),axis=0)
_video_db_future = np.concatenate((_video_db_future, per_video_db_future),axis=0)
_video_db_future_input = np.concatenate((_video_db_future_input, per_video_db_future_input),axis=0)
return _video_db[1:,:,:], _video_db_future[1:,:,:], _video_db_future_input[1:,:,:]
else:
_video_db_tar = np.zeros((1,max_encoder_seq_length,num_encoder_tokens))
_video_db_future_tar = np.zeros((1,max_encoder_seq_length,num_encoder_tokens))
_video_db_future_input_tar = np.zeros((1,max_encoder_seq_length,num_encoder_tokens))
_video_db_oth = np.zeros((num_user-1,1,max_encoder_seq_length,num_encoder_tokens))
_video_db_future_oth = np.zeros((num_user-1,1,max_encoder_seq_length,num_encoder_tokens))
_video_db_future_input_oth = np.zeros((num_user-1,1,max_encoder_seq_length,num_encoder_tokens))
for _video_ind in datadb.keys():
# for each video, pick out a random target user, split target user and other users
_target_user = np.random.randint(0,num_user)
per_video_db = np.stack((datadb[_video_ind]['x'],datadb[_video_ind]['y'],datadb[_video_ind]['z']),axis=-1)
per_video_db = per_video_db[:,per_video_db.shape[1]-per_video_db.shape[1]/fps/max_encoder_seq_length*fps*max_encoder_seq_length:,:] #cut head to make dividable by seconds
per_video_db = np.reshape(per_video_db,(per_video_db.shape[0],per_video_db.shape[1]/fps,num_encoder_tokens))
per_video_db_tar = per_video_db[_target_user,:][np.newaxis,:,:]
per_video_db_oth = np.delete(per_video_db,_target_user,axis=0)
per_video_db_tar, per_video_db_future_tar, per_video_db_future_input_tar = reshape2second_stacks(per_video_db_tar,collapse_user=True)
per_video_db_oth, per_video_db_future_oth, per_video_db_future_input_oth = reshape2second_stacks(per_video_db_oth,collapse_user=False)
_video_db_tar = np.concatenate((_video_db_tar, per_video_db_tar),axis=0)
_video_db_future_tar = np.concatenate((_video_db_future_tar, per_video_db_future_tar),axis=0)
_video_db_future_input_tar = np.concatenate((_video_db_future_input_tar, per_video_db_future_input_tar),axis=0)
_video_db_oth = np.concatenate((_video_db_oth, per_video_db_oth),axis=1)
_video_db_future_oth = np.concatenate((_video_db_future_oth, per_video_db_future_oth),axis=1)
_video_db_future_input_oth = np.concatenate((_video_db_future_input_oth, per_video_db_future_input_oth),axis=1)
return _video_db_tar[1:,:,:], _video_db_future_tar[1:,:,:], _video_db_future_input_tar[1:,:,:], \
_video_db_oth[:,1:,:,:], _video_db_future_oth[:,1:,:,:], _video_db_future_input_oth[:,1:,:,:]
## only use one video for now
# _video_ind = 0
# per_video_db = np.stack((datadb[_video_ind]['x'],datadb[_video_ind]['y'],datadb[_video_ind]['z']),axis=-1)
# per_video_db = per_video_db[:,per_video_db.shape[1]-per_video_db.shape[1]/fps*fps:,:] #cut head to make dividable by seconds
# per_video_db = np.reshape(per_video_db,(per_video_db.shape[0],per_video_db.shape[1]/fps,90))
def get_gt_target_xyz(y):
"""get gt mean var"""
target_x = y[:,:,0::3]
target_y = y[:,:,1::3]
target_z = y[:,:,2::3]
gt_mean_x = np.mean(target_x, axis=2)[:,:,np.newaxis]
gt_var_x = np.var(target_x, axis=2)[:,:,np.newaxis]
gt_mean_y = np.mean(target_y, axis=2)[:,:,np.newaxis]
gt_var_y = np.var(target_y, axis=2)[:,:,np.newaxis]
gt_mean_z = np.mean(target_z, axis=2)[:,:,np.newaxis]
gt_var_z = np.var(target_z, axis=2)[:,:,np.newaxis]
return np.concatenate((gt_mean_x,gt_mean_y,gt_mean_z,gt_var_x,gt_var_y,gt_var_z),axis=2)
def get_gt_target_row_col(y):
"""get gt mean var"""
target_row = y[:,:,0::2]
target_col = y[:,:,1::2]
gt_mean_row = np.mean(target_row, axis=2)[:,:,np.newaxis]
gt_var_row = np.var(target_row, axis=2)[:,:,np.newaxis]
gt_mean_col = np.mean(target_col, axis=2)[:,:,np.newaxis]
gt_var_col = np.var(target_col, axis=2)[:,:,np.newaxis]
return np.concatenate((gt_mean_row,gt_mean_col,gt_var_row,gt_var_col),axis=2)
def get_gt_target_72(y):
"""get gt mean var"""
for i in range(72):
target_code = y[:,:,i::72]
gt_mean = np.mean(target_code,axis=2)[:,:,np.newaxis]
#gt_var = np.var(target_code,axis=2)[:,:,np.newaxis]
if i == 0:
gt = gt_mean
else:
gt = np.concatenate((gt,gt_mean),axis=2)
return gt
#assign data
## only use one video for now
# encoder_input_data = per_video_db[:,100:110,:]
# temp = per_video_db[:,110:120,:].copy()
# temp1 = np.zeros_like(temp)
# temp1[:,1:,:] = temp[:,:-1,:].copy()
# temp1[:,0,:] = encoder_input_data[:,-1,:]
_video_db,_video_db_future,_video_db_future_input = get_data(datadb,pick_user=False)
num_samples = _video_db.shape[0] # Number of samples to train on.
num_testing_sample = 1000
print(_video_db.shape)
#encoder_input_data = _video_db[:-num_testing_sample,:,:]######################
encoder_input_data = get_gt_target_xyz(_video_db)[:-num_testing_sample,:,:]
decoder_target_data = get_gt_target_xyz(_video_db_future)[:-num_testing_sample,:,:]
decoder_input_data = get_gt_target_xyz(_video_db_future_input)[:-num_testing_sample,:,:]
#decoder_target_data = get_gt_target_row_col(_video_db_future)[:-num_testing_sample,:,:]
#decoder_input_data = get_gt_target_row_col(_video_db_future_input)[:-num_testing_sample,:,:]
#decoder_target_data = get_gt_target_72(_video_db_future)[:-num_testing_sample,:,:]
#decoder_input_data = get_gt_target_72(_video_db_future_input)[:-num_testing_sample,:,:]
### ====================Graph def====================
# Define an input sequence and process it.
encoder_inputs = Input(shape=(None, 6))
encoder = LSTM(latent_dim, return_state=True)
encoder_outputs, state_h, state_c = encoder(encoder_inputs)
# We discard `encoder_outputs` and only keep the states.
encoder_states = [state_h, state_c]
# Set up the decoder, using `encoder_states` as initial state.
decoder_inputs = Input(shape=(None, num_decoder_tokens))
# We set up our decoder to return full output sequences,
# and to return internal states as well. We don't use the
# return states in the training model, but we will use them in inference.
decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_inputs,
initial_state=encoder_states)
decoder_dense = Dense(num_decoder_tokens,activation='tanh')
decoder_outputs = decoder_dense(decoder_outputs)
### ====================Training====================
model_checkpoint = ModelCheckpoint('fov_s2s_noteacherforcing_epoch{epoch:02d}-{val_loss:.4f}.h5', monitor='val_loss', save_best_only=True)
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=3, min_lr=1e-6)
stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto')
# Define the model that will turn
# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
# Run training
model.compile(optimizer='Adam', loss='mean_squared_error',metrics=['accuracy'])
model.fit([encoder_input_data, decoder_input_data], decoder_target_data,
batch_size=batch_size,
epochs=epochs,
validation_split=0.2,
shuffle=True,
callbacks=[model_checkpoint, reduce_lr, stopping])
# Save model
model.save('fov_s2s_tanh.h5')
print("aaaaaaaaaaaaaaaa")
### ====================Testing====================
# Next: inference mode (sampling).
# Here's the drill:
# 1) encode input and retrieve initial decoder state
# 2) run one step of decoder with this initial state
# and a "start of sequence" token as target.
# Output will be the next target token
# 3) Repeat with the current target token and current states
# Define sampling models
encoder_model = Model(encoder_inputs, encoder_states)
decoder_state_input_h = Input(shape=(latent_dim,))
decoder_state_input_c = Input(shape=(latent_dim,))
decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]
decoder_outputs, state_h, state_c = decoder_lstm(
decoder_inputs, initial_state=decoder_states_inputs)
decoder_states = [state_h, state_c]
decoder_outputs = decoder_dense(decoder_outputs)
decoder_model = Model(
[decoder_inputs] + decoder_states_inputs,
[decoder_outputs] + decoder_states)
# from keras.models import load_model
# temp = load_model('fov_s2s.h5')
def decode_sequence_fov(input_seq):
# Encode the input as state vectors.
states_value = encoder_model.predict(get_gt_target_xyz(input_seq))
last_location = input_seq[0,-1,:][np.newaxis,np.newaxis,:]
last_mu_var = get_gt_target_xyz(last_location) ########xyz
target_seq = last_mu_var
# target_seq = np.zeros((1, 1, num_decoder_tokens))
# Sampling loop for a batch of sequences
# (to simplify, here we assume a batch of size 1).
decoded_sentence = []
for ii in range(cfg.predict_step):
output_tokens, h, c = decoder_model.predict(
[target_seq] + states_value)
decoded_sentence+=[output_tokens]
# # Update the target sequence (of length 1).
# target_seq = np.zeros((1, 1, num_decoder_tokens))
target_seq = output_tokens
# Update states
states_value = [h, c]
return decoded_sentence
gt_sentence_list = []
decoded_sentence_list = []
for seq_index in range(num_samples-num_testing_sample,num_samples-num_testing_sample+100):
# Take one sequence (part of the training set)
# for trying out decoding.
input_seq = _video_db[seq_index: seq_index + 1,:,:]
#input_seq = get_gt_xyz
decoded_sentence = decode_sequence_fov(input_seq)
decoded_sentence_list+=[decoded_sentence]
gt_sentence = _video_db_future[seq_index: seq_index + 1,:,:]
gt_sentence_list+=[gt_sentence]
decoder_target = get_gt_target_xyz(gt_sentence)##############xyz
# print('-')
# print('Decoded sentence - decoder_target:', np.squeeze(np.array(decoded_sentence))[:,:3]-np.squeeze(decoder_target)[:,:3])
pickle.dump(decoded_sentence_list,open('decoded_sentence.p','wb'))
pickle.dump(gt_sentence_list,open('gt_sentence_list.p','wb'))
print('Testing finished!')