-
Notifications
You must be signed in to change notification settings - Fork 0
/
.env
95 lines (91 loc) · 3.52 KB
/
.env
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
function runc() {
local usage="runc <lang> <local|semantica|exjobb8> [demo|eval] [extra opts]"
# Pretty colours
local L_RED="\e[91m"
local RES="\e[0m"
local GREEN="\e[32m"
local CYAN="\e[95m"
# check language
case "$1" in
sv)
local language="$1"
;;
en)
local language="$1"
;;
*)
echo $usage
printf "${L_RED}Error! unknown or missing language: %s\n$RES" $1
return 1
esac
# host
case "$2" in
local)
: "${DATA_PATH:?You need to set DATA_PATH non-empty}"
local corpus_path="file:${DATA_PATH}/wikipedia-corpus-herd/$1"
local relation_config_path="file:${DATA_PATH}/target_relations.tsv"
local wikidata_path="file:${DATA_PATH}/wikidata"
local temp_data_path="file:${DATA_PATH}/prometheus-data/0.8.0/relation_model"
local eval_files="file:${DATA_PATH}/eval_files/place_of_birth.json ../data/eval_files/institution.json"
local word2vec_path="file:${DATA_PATH}/word2vec/$1"
export SPARK_MASTER="local[*]"
;;
semantica)
local corpus_path="hdfs:/user/ine11ega/herd/$1"
local relation_config_path="hdfs:/user/ine11ega/configs/target_relations_$1.tsv"
local wikidata_path="hdfs:/user/marcusk/wikipedia-pipeline/mul/parsed-wikidata"
local temp_data_path="hdfs:/user/ine11ega/relation_model"
local eval_files="hdfs:/user/dat11al1/eval-place_of_birth.json.txt hdfs:/user/dat11al1/eval_institution.json"
local word2vec_path="hdfs:/user/ine11ega/word2vec/$1"
;;
aws)
local corpus_path="s3://sony-prometheus-data/herd/$1"
local relation_config_path="s3://sony-prometheus-data/configs/target_relations_$1.tsv"
local wikidata_path="s3://sony-prometheus-data/wikidata"
local temp_data_path="s3://sony-prometheus-data/relation_model"
local eval_files="s3://sony-prometheus-data/google-evaluation/eval-place_of_birth.json s3://sony-prometheus-data/google-evaluation/eval_institution.json"
local word2vec_path="s3://sony-prometheus-data/word2vec/$1"
;;
exjobb8)
local corpus_path="hdfs:/user/antons/wikipedia-pipeline/$1/herd"
local relation_config_path="hdfs:/user/ine11ega/target_relations.tsv"
local wikidata_path="hdfs:/home/erikaxel/wikidata"
local temp_data_path="file:/home/erikaxel/relation_model-pos"
local eval_files="hdfs:/user/dat11al1/eval-place_of_birth.json.txt hdfs:/user/dat11al1/eval_institution.json"
local word2vec_path="file:/home/erikaxel/word2vec/$1"
;;
*)
echo $usage
printf "${L_RED}Error! unknown or missing target: %s\n$RES" $2
return 1
esac
# mode
case "$3" in
eval)
local extra_opts=${@:4}
local args=" --model-evaluation-files $eval_files ${extra_opts} $corpus_path $relation_config_path $wikidata_path $temp_data_path $word2vec_path"
if [ "$2" = "local" ]; then
local cmd="sbt \"run $args\""
printf "${GREEN}%s\n${RES}" $cmd
eval $cmd
else
local cmd="./scripts/runc/run-cloud.sh ./scripts/runc/$2 $args"
printf "${GREEN}%s\n${RES}" $cmd
eval $cmd
fi
;;
*)
local extra_opts=${@:3}
local args="${extra_opts} $corpus_path $relation_config_path $wikidata_path $temp_data_path $word2vec_path"
if [ "$2" = "local" ]; then
local cmd="sbt \"run $args\""
printf "${GREEN}%s\n${RES}" $cmd
eval $cmd
else
local cmd="./scripts/runc/run-cloud.sh ./scripts/runc/$2 $args"
printf "${GREEN}%s\n${RES}" $cmd
eval $cmd
fi
;;
esac
}