Skip to content

Commit

Permalink
removes unused and useless code
Browse files Browse the repository at this point in the history
  • Loading branch information
sadit committed Mar 20, 2024
1 parent 0265316 commit 93ab1a4
Show file tree
Hide file tree
Showing 3 changed files with 0 additions and 48 deletions.
44 changes: 0 additions & 44 deletions common.jl
Original file line number Diff line number Diff line change
Expand Up @@ -82,36 +82,6 @@ function create_pca_model(dist, file::String; nbits::Int)
fit(PCAProjection, A, dim), SqL2Distance(), "PCA-$(dim)"
end

function create_binperms_model(dist, file::String; nbits::Int, nrefs::Int=2048)
A = h5open(file) do f
X = f["emb"]
m, n = size(X)
n2 = min(10^6, n ÷ 3)
X[:, 1:n2]
end

@show size(A) typeof(A)
refs = let
C = fft(dist, MatrixDatabase(A), nrefs) # select `nrefs` distant elements -- kcenters using farthest first traversal
MatrixDatabase(A[:, C.centers])
end

fit(BinPerms, dist, refs, nbits), BinaryHammingDistance(), "BinPerms-$nbits"
end

function create_heh_model(dist, file::String; nbits::Int)
A = h5open(file) do f
X = f["emb"]
m, n = size(X)
n2 = 2^15
X[:, 1:n2]
end

@show size(A) typeof(A)
#fit(highentropyhyperplanes, dist, matrixdatabase(a), nbits; sample_for_hyperplane_selection=2^16, k=4092, k2=1024), binaryhammingdistance(), "highentropyhyperplanes-$nbits"
fit(HighEntropyHyperplanes, dist, MatrixDatabase(A), nbits; minent=0.5,
sample_for_hyperplane_selection=2^13), BinaryHammingDistance(), "HighEntropyHyperplanes-$nbits"
end


function predict_h5(model::Union{PCAProjection,GaussianRandomProjection}, file::String; nbits, block::Int=10^5)
Expand All @@ -129,17 +99,3 @@ function predict_h5(model::Union{PCAProjection,GaussianRandomProjection}, file::
end
end

function predict_h5(model::Union{BinPerms,HighEntropyHyperplanes}, file::String; nbits, block::Int=10^5)
h5open(file) do f
X = f["emb"]
m, n = size(X)
B = Matrix{UInt64}(undef, nbits ÷ 64, n)
for group in Iterators.partition(1:n, block)
@info "encoding $group of $n -- $(Dates.now())"
B[:, group] .= predict(model, MatrixDatabase(X[:, group])).matrix
end

StrideMatrixDatabase(B)
end
end

1 change: 0 additions & 1 deletion eval.jl
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ function evaluate_results(gfile, resultfiles, k)
end

function print_results(f, D, gfile, files, task, dbsize)
println(f, gfile => files)
show(f, "text/plain", gfile => files); println(f)
show(f, "text/plain", D); println(f)
p = lineplot(D.recall; ylim=(0, 1), title=String(D.algo[1]), ylabel="recall", xlabel="$(D.params[1]) to $(D.params[end])")
Expand Down
3 changes: 0 additions & 3 deletions task3.jl
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,9 @@ function task3(;

mkpath(outdir)
dist = NormalizedCosineDistance() # 1 - dot(·, ·)
#nbits = 8 * 4 * 128 # memory eq to 128 fp32
nbits = 8 * 4 * 128 # memory eq to 128 fp32
#model, dist_proj, nick = create_binperms_model(dist, dfile; nbits)
#model, dist_proj, nick = create_rp_model(dist, dfile; nbits)
modelingtime = @elapsed model, dist_proj, nick = create_pca_model(dist, dfile; nbits)
#modelingtime = @elapsed model, dist_proj, nick = create_heh_model(dist, dfile; nbits)
encdatabasetime = @elapsed db = predict_h5(model, dfile; nbits)
encqueriestime = @elapsed queries = predict_h5(model, qfile; nbits)

Expand Down

0 comments on commit 93ab1a4

Please sign in to comment.