#!/bin/bash
# Applies sorting and splitting on all files.

set -eou pipefail

main() {
    ./lib/languages_update.py
    local -r REGEX_STRING=$(./lib/common_characters.py)
    for TSV in tsv/*.tsv; do
        # Explicitly uses byte-wise comparison for sorting
        # rather than a locale-dependent comparison.
        LC_ALL=C sort -u -o "${TSV}" "${TSV}"
        ./lib/split.py --regex_string="${REGEX_STRING}" "${TSV}" 
    done
    ./lib/generate_summary.py
}

main
