#!/usr/bin/env python3

import transformers
import sys
import os

model_name = os.environ["MODEL_NAME"]
tokenizer = transformers.MarianTokenizer.from_pretrained(model_name)
input_path = sys.argv[1]
output_path = sys.argv[2]

with open(input_path) as input_file, open(output_path, "w") as output_file:
    for line in input_file:
        ids = tokenizer.encode(line.strip())
        output_file.write(" ".join(map(str, ids)))
        output_file.write("\n")
