From 56f74963d6856c21959660a1cdb5e6daba8e761c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20M=C3=BCller?= Date: Mon, 27 Jan 2020 16:57:15 +0100 Subject: [PATCH] do not fail if no tokenization step --- source/embed.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/embed.py b/source/embed.py index c775ffce..b0a37221 100644 --- a/source/embed.py +++ b/source/embed.py @@ -364,15 +364,21 @@ def EmbedMmap(fname, dim=1024, dtype=np.float32, verbose=False): with tempfile.TemporaryDirectory() as tmpdir: ifname = '' # stdin will be used + tok_fname = os.path.join(tmpdir, 'tok') + if args.token_lang != '--': - tok_fname = os.path.join(tmpdir, 'tok') + Token(ifname, tok_fname, lang=args.token_lang, romanize=True if args.token_lang == 'el' else False, lower_case=True, gzip=False, verbose=args.verbose, over_write=False) - ifname = tok_fname + else: + with open(tok_fname, "w") as ofname: + for line in sys.stdin: + ofname.write(line) + ifname = tok_fname if args.bpe_codes: bpe_fname = os.path.join(tmpdir, 'bpe')