-
Notifications
You must be signed in to change notification settings - Fork 3
/
feature_axis.py
205 lines (155 loc) · 8.07 KB
/
feature_axis.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
""" module of functions related to discovering feature axis """
import time
import numpy as np
import sklearn.linear_model as linear_model
def find_feature_axis(z, y, method='linear', **kwargs_model):
"""
function to find axis in the latent space that is predictive of feature vectors
:param z: vectors in the latent space, shape=(num_samples, num_latent_vector_dimension)
:param y: feature vectors, shape=(num_samples, num_features)
:param method: one of ['linear', 'logistic'], or a sklearn.linear_model object, (eg. sklearn.linear_model.ElasticNet)
:param kwargs_model: parameters specific to a sklearn.linear_model object, (eg., penalty=’l2’)
:return: feature vectors, shape = (num_latent_vector_dimension, num_features)
"""
if method == 'linear':
model = linear_model.LinearRegression(**kwargs_model)
model.fit(z, y)
elif method == 'tanh':
def arctanh_clip(y):
return np.arctanh(np.clip(y, np.tanh(-3), np.tanh(3)))
model = linear_model.LinearRegression(**kwargs_model)
model.fit(z, arctanh_clip(y))
else:
raise Exception('method has to be one of ["linear", "tanh"]')
return model.coef_.transpose()
def normalize_feature_axis(feature_slope):
"""
function to normalize the slope of features axis so that they have the same length
:param feature_slope: array of feature axis, shape = (num_latent_vector_dimension, num_features)
:return: same shape of input
"""
feature_direction = feature_slope / np.linalg.norm(feature_slope, ord=2, axis=0, keepdims=True)
return feature_direction
def disentangle_feature_axis(feature_axis_target, feature_axis_base, yn_base_orthogonalized=False):
"""
make feature_axis_target orthogonal to feature_axis_base
:param feature_axis_target: features axes to decorrerelate, shape = (num_dim, num_feature_0)
:param feature_axis_base: features axes to decorrerelate, shape = (num_dim, num_feature_1))
:param yn_base_orthogonalized: True/False whether the feature_axis_base is already othogonalized
:return: feature_axis_decorrelated, shape = shape = (num_dim, num_feature_0)
"""
# make sure this funciton works to 1D vector
if len(feature_axis_target.shape) == 0:
yn_single_vector_in = True
feature_axis_target = feature_axis_target[:, None]
else:
yn_single_vector_in = False
# if already othogonalized, skip this step
if yn_base_orthogonalized:
feature_axis_base_orthononal = orthogonalize_vectors(feature_axis_base)
else:
feature_axis_base_orthononal = feature_axis_base
# orthogonalize every vector
feature_axis_decorrelated = feature_axis_target + 0
num_dim, num_feature_0 = feature_axis_target.shape
num_dim, num_feature_1 = feature_axis_base_orthononal.shape
for i in range(num_feature_0):
for j in range(num_feature_1):
feature_axis_decorrelated[:, i] = orthogonalize_one_vector(feature_axis_decorrelated[:, i],
feature_axis_base_orthononal[:, j])
# make sure this funciton works to 1D vector
if yn_single_vector_in:
result = feature_axis_decorrelated[:, 0]
else:
result = feature_axis_decorrelated
return result
def disentangle_feature_axis_by_idx(feature_axis, idx_base=None, idx_target=None, yn_normalize=True):
"""
disentangle correlated feature axis, make the features with index idx_target orthogonal to
those with index idx_target, wrapper of function disentangle_feature_axis()
:param feature_axis: all features axis, shape = (num_dim, num_feature)
:param idx_base: index of base features (1D numpy array), to which the other features will be orthogonal
:param idx_target: index of features to disentangle (1D numpy array), which will be disentangled from
base features, default to all remaining features
:param yn_normalize: True/False to normalize the results
:return: disentangled features, shape = feature_axis
"""
(num_dim, num_feature) = feature_axis.shape
# process default input
if idx_base is None or len(idx_base) == 0: # if None or empty, do nothing
feature_axis_disentangled = feature_axis
else: # otherwise, disentangle features
if idx_target is None: # if None, use all remaining features
idx_target = np.setdiff1d(np.arange(num_feature), idx_base)
feature_axis_target = feature_axis[:, idx_target] + 0
feature_axis_base = feature_axis[:, idx_base] + 0
feature_axis_base_orthogonalized = orthogonalize_vectors(feature_axis_base)
feature_axis_target_orthogonalized = disentangle_feature_axis(
feature_axis_target, feature_axis_base_orthogonalized, yn_base_orthogonalized=True)
feature_axis_disentangled = feature_axis + 0 # holder of results
feature_axis_disentangled[:, idx_target] = feature_axis_target_orthogonalized
feature_axis_disentangled[:, idx_base] = feature_axis_base_orthogonalized
# normalize output
if yn_normalize:
feature_axis_out = normalize_feature_axis(feature_axis_disentangled)
else:
feature_axis_out = feature_axis_disentangled
return feature_axis_out
def orthogonalize_one_vector(vector, vector_base):
"""
tool function, adjust vector so that it is orthogonal to vector_base (i.e., vector - its_projection_on_vector_base )
:param vector0: 1D array
:param vector1: 1D array
:return: adjusted vector1
"""
return vector - np.dot(vector, vector_base) / np.dot(vector_base, vector_base) * vector_base
def orthogonalize_vectors(vectors):
"""
tool function, adjust vectors so that they are orthogonal to each other, takes O(num_vector^2) time
:param vectors: vectors, shape = (num_dimension, num_vector)
:return: orthorgonal vectors, shape = (num_dimension, num_vector)
"""
vectors_orthogonal = vectors + 0
num_dimension, num_vector = vectors.shape
for i in range(num_vector):
for j in range(i):
vectors_orthogonal[:, i] = orthogonalize_one_vector(vectors_orthogonal[:, i], vectors_orthogonal[:, j])
return vectors_orthogonal
def plot_feature_correlation(feature_direction, feature_name=None):
import matplotlib.pyplot as plt
len_z, len_y = feature_direction.shape
if feature_name is None:
feature_name = range(len_y)
feature_correlation = np.corrcoef(feature_direction.transpose())
c_lim_abs = np.max(np.abs(feature_correlation))
plt.pcolormesh(np.arange(len_y+1), np.arange(len_y+1), feature_correlation,
cmap='coolwarm', vmin=-c_lim_abs, vmax=+c_lim_abs)
plt.gca().invert_yaxis()
plt.colorbar()
# plt.axis('square')
plt.xticks(np.arange(len_y) + 0.5, feature_name, fontsize='x-small', rotation='vertical')
plt.yticks(np.arange(len_y) + 0.5, feature_name, fontsize='x-small')
plt.show()
def plot_feature_cos_sim(feature_direction, feature_name=None):
"""
plot cosine similarity measure of vectors
:param feature_direction: vectors, shape = (num_dimension, num_vector)
:param feature_name: list of names of features
:return: cosines similarity matrix, shape = (num_vector, num_vector)
"""
import matplotlib.pyplot as plt
from sklearn.metrics.pairwise import cosine_similarity
len_z, len_y = feature_direction.shape
if feature_name is None:
feature_name = range(len_y)
feature_cos_sim = cosine_similarity(feature_direction.transpose())
c_lim_abs = np.max(np.abs(feature_cos_sim))
plt.pcolormesh(np.arange(len_y+1), np.arange(len_y+1), feature_cos_sim,
vmin=-c_lim_abs, vmax=+c_lim_abs, cmap='coolwarm')
plt.gca().invert_yaxis()
plt.colorbar()
# plt.axis('square')
plt.xticks(np.arange(len_y) + 0.5, feature_name, fontsize='x-small', rotation='vertical')
plt.yticks(np.arange(len_y) + 0.5, feature_name, fontsize='x-small')
plt.show()
return feature_cos_sim