#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (C) 2020 T. Zachary Laine
#
# Distributed under the Boost Software License, Version 1.0. (See
# accompanying file LICENSE_1_0.txt or copy at
# http://www.boost.org/LICENSE_1_0.txt)

from generate_unicode_normalization_data import cccs
from generate_unicode_normalization_data import expand_decomp_canonical
from generate_unicode_normalization_data import get_decompositions
from generate_unicode_collation_data import get_frac_uca_cet
from generate_unicode_collation_data import ce_to_cpp

import re

perf_test_form = decls = '''\
// Copyright (C) 2020 T. Zachary Laine
//
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

// Warning! This file is autogenerated.
#include <boost/text/collate.hpp>
#include <boost/text/collation_table.hpp>

#include <benchmark/benchmark.h>

boost::text::detail::collation_trie_t const & trie()
{{
    static auto const retval = boost::text::detail::make_default_trie();
    return retval;
}}

boost::text::collation_table const & table()
{{
    static auto const retval = boost::text::default_collation_table();
    return retval;
}}

{0}

BENCHMARK_MAIN()
'''

relative_collation_tests_form = '''\
// Copyright (C) 2020 T. Zachary Laine
//
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

// Warning! This file is autogenerated.
#include "collation_tests.hpp"

#include <algorithm>

#include <gtest/gtest.h>


std::array<uint32_t, {1}> const g_cps = {{{{
{0}
}}}};

std::array<std::pair<uint32_t, uint32_t>, {3}> const g_cp_ranges = {{{{
{2}
}}}};

TEST(collation, relative_{4}_{5}_fcc)
{{
    std::vector<uint32_t> prev_un_norm;
    std::vector<uint32_t> prev_cps;
    boost::text::text_sort_key prev_key;
    std::vector<uint32_t> curr_un_norm;
    std::vector<uint32_t> curr_cps;
    boost::text::text_sort_key curr_key;

    bool first = true;

    std::pair<uint32_t, uint32_t> prev_r;

    for (auto r : g_cp_ranges) {{
        curr_un_norm.assign(
            g_cps.begin() + r.first, g_cps.begin() + r.second);

        curr_cps.clear();
        boost::text::normalize<boost::text::nf::fcc>(
            curr_un_norm, std::back_inserter(curr_cps));

        curr_key = boost::text::collation_sort_key(
            curr_cps.begin(),
            curr_cps.end(),
            table(),
            boost::text::collation_strength::identical,
            boost::text::case_first::off,
            boost::text::case_level::off,
            boost::text::variable_weighting::{4});

        if (!first) {{
            EXPECT_LE(compare(prev_key, curr_key), 0)
                << "prev un-norm cps: " << ce_dumper(prev_un_norm)
                << "prev_cps (FCC):   " << ce_dumper(prev_cps)
                << "prev_key:         " << ce_dumper(prev_key) << "\\n"
                << "curr un-norm cps: " << ce_dumper(curr_un_norm)
                << "curr_cps (FCC):   " << ce_dumper(curr_cps)
                << "curr_key:         " << ce_dumper(curr_key) << "\\n"
            ;

            std::string prev = boost::text::to_string(prev_cps);
            std::string curr = boost::text::to_string(curr_cps);
            auto const prev_32 = boost::text::as_utf32(prev);
            auto const curr_32 = boost::text::as_utf32(curr);
            EXPECT_LE(
                boost::text::collate(
                    prev_32.begin(),
                    prev_32.end(),
                    curr_32.begin(),
                    curr_32.end(),
                    table(),
                    boost::text::collation_strength::identical,
                    boost::text::case_first::off,
                    boost::text::case_level::off,
                    boost::text::variable_weighting::{4}),
                0)
                << "prev un-norm cps: " << ce_dumper(prev_un_norm)
                << "prev_cps (FCC):   " << ce_dumper(prev_cps)
                << "prev_key:         " << ce_dumper(prev_key) << "\\n"
                << "curr un-norm cps: " << ce_dumper(curr_un_norm)
                << "curr_cps (FCC):   " << ce_dumper(curr_cps)
                << "curr_key:         " << ce_dumper(curr_key) << "\\n"
            ;
        }}

        std::swap(curr_un_norm, prev_un_norm);
        std::swap(curr_cps, prev_cps);
        std::swap(curr_key, prev_key);

        first = false;

        prev_r = r;
    }}
}}

TEST(collation, relative_{4}_{5}_nfd)
{{
    std::vector<uint32_t> prev_un_norm;
    std::vector<uint32_t> prev_cps;
    boost::text::text_sort_key prev_key;
    std::vector<uint32_t> curr_un_norm;
    std::vector<uint32_t> curr_cps;
    boost::text::text_sort_key curr_key;

    bool first = true;

    std::pair<uint32_t, uint32_t> prev_r;

    for (auto r : g_cp_ranges) {{
        curr_un_norm.assign(
            g_cps.begin() + r.first, g_cps.begin() + r.second);

        curr_cps.clear();
        boost::text::normalize<boost::text::nf::d>(
            curr_un_norm, std::back_inserter(curr_cps));

        curr_key = boost::text::collation_sort_key(
            curr_cps.begin(),
            curr_cps.end(),
            table(),
            boost::text::collation_strength::identical,
            boost::text::case_first::off,
            boost::text::case_level::off,
            boost::text::variable_weighting::{4});

        if (!first) {{
            EXPECT_LE(compare(prev_key, curr_key), 0)
                << "prev un-norm cps: " << ce_dumper(prev_un_norm)
                << "prev_cps (NFD):   " << ce_dumper(prev_cps)
                << "prev_key:         " << ce_dumper(prev_key) << "\\n"
                << "curr un-norm cps: " << ce_dumper(curr_un_norm)
                << "curr_cps (NFD):   " << ce_dumper(curr_cps)
                << "curr_key:         " << ce_dumper(curr_key) << "\\n"
            ;

            std::string prev = boost::text::to_string(prev_cps);
            std::string curr = boost::text::to_string(curr_cps);
            auto const prev_32 = boost::text::as_utf32(prev);
            auto const curr_32 = boost::text::as_utf32(curr);
            EXPECT_LE(
                boost::text::collate(
                    prev_32.begin(),
                    prev_32.end(),
                    curr_32.begin(),
                    curr_32.end(),
                    table(),
                    boost::text::collation_strength::identical,
                    boost::text::case_first::off,
                    boost::text::case_level::off,
                    boost::text::variable_weighting::{4}),
                0)
                << "prev un-norm cps: " << ce_dumper(prev_un_norm)
                << "prev_cps (NFD):   " << ce_dumper(prev_cps)
                << "prev_key:         " << ce_dumper(prev_key) << "\\n"
                << "curr un-norm cps: " << ce_dumper(curr_un_norm)
                << "curr_cps (NFD):   " << ce_dumper(curr_cps)
                << "curr_key:         " << ce_dumper(curr_key) << "\\n"
            ;
        }}

        std::swap(curr_un_norm, prev_un_norm);
        std::swap(curr_cps, prev_cps);
        std::swap(curr_key, prev_key);

        first = false;

        prev_r = r;
    }}
}}
'''

def indices_to_list(indices, all_cps):
    return all_cps[indices[0]:indices[1]]

def generate_lookup_perf_test(ducet):
    chunk_size = 50
    chunks_per_file = 100

    chunk_arrays = []

    chunk = 0
    i = 0
    cps = []
    cp_ranges = []
    for k,v in sorted(ducet.items()):
        cp_ranges.append((len(cps), len(cps) + len(k)))
        cps += list(k)
        i += 1
        if i == chunk_size:
            chunk_arrays.append((cps, cp_ranges))
            chunk += 1
            i = 0
            cps = []
            cp_ranges = []

    chunk_idx = 0
    lines = ''
    for i in range(len(chunk_arrays)):
        if i != 0 and i % chunks_per_file == 0:
            cpp_file = open('collation_element_lookup_perf_{0:03}.cpp'.format(chunk_idx), 'w')
            cpp_file.write(perf_test_form.format(lines))
            chunk_idx += 1
            lines = ''
        cps = chunk_arrays[i][0]
        cp_ranges = chunk_arrays[i][1]
        lines += '''\
uint32_t cps_{0:03}[] = {{
{1}
}};

void BM_collation_element_lookup_{0:03}(benchmark::State & state)
{{
    while (state.KeepRunning()) {{
'''.format(i, ', '.join(map(lambda x: type(x) == str and '0x' + x or hex(x), cps)), len(cps))
        for first,last in cp_ranges:
            lines += '''\
            benchmark::DoNotOptimize(trie().longest_match(cps_{0:03} + {1}, cps_{0:03} + {2}));
'''.format(i, first, last)
        lines += '''\
    }}
}}
BENCHMARK(BM_collation_element_lookup_{0:03});

'''.format(i)

    cpp_file = open('collation_element_lookup_perf_{0:03}.cpp'.format(chunk_idx), 'w')
    cpp_file.write(perf_test_form.format(lines))

def generate_collation_perf_test(ducet):
    chunk_size = 100
    chunks_per_file = 100

    chunk_arrays = []

    chunk = 0
    i = 0
    cps = []
    cp_ranges = []
    for k,v in sorted(ducet.items()):
        cp_ranges.append((len(cps), len(cps) + len(k)))
        cps += list(k)
        i += 1
        if i == chunk_size:
            chunk_arrays.append((cps, cp_ranges))
            chunk += 1
            i = 0
            cps = []
            cp_ranges = []

    chunk_idx = 0
    lines = ''
    for i in range(len(chunk_arrays)):
        if i != 0 and i % chunks_per_file == 0:
            cpp_file = open('collation_perf_{0:03}.cpp'.format(chunk_idx), 'w')
            cpp_file.write(perf_test_form.format(lines))
            chunk_idx += 1
            lines = ''
        cps = chunk_arrays[i][0]
        cp_ranges = chunk_arrays[i][1]
        lines += '''\
uint32_t cps_{0:03}[] = {{
{1}
}};

void BM_collation_{0:03}(benchmark::State & state)
{{
    while (state.KeepRunning()) {{
'''.format(i, ', '.join(map(lambda x: type(x) == str and '0x' + x or hex(x), cps)), len(cps))
        lines += '''\
        benchmark::DoNotOptimize(boost::text::collation_sort_key(cps_{0:03}, cps_{0:03} + {2}, table(), boost::text::collation_strength::quaternary, boost::text::case_first::off, boost::text::case_level::off, boost::text::variable_weighting::shifted));
'''.format(i, cp_ranges[0][0], cp_ranges[-1][1])
        lines += '''\
    }}
}}
BENCHMARK(BM_collation_{0:03});

'''.format(i)

    cpp_file = open('collation_perf_{0:03}.cpp'.format(chunk_idx), 'w')
    cpp_file.write(perf_test_form.format(lines))

collation_elements_regex = re.compile(r'\[([ |0123456789ABCDEF]+)\]')

def generate_relative_collation_tests(filename, weighting):
    lines = open(filename, 'r').readlines()

    all_cps = []
    all_ranges = []
    chunk_idx = 0
    line_idx = 0 
    for line in lines:
        if line_idx == 25000:
            cps_string = ', '.join(map(lambda x: '0x' + x, all_cps))
            ranges_string = ', '.join(map(lambda x: '{{{}, {}}}'.format(x[0], x[1]), all_ranges))
            cpp_file = open('relative_collation_test_{0}_{1}.cpp'.format(weighting, chunk_idx), 'w')
            cpp_file.write(relative_collation_tests_form.format(cps_string, len(all_cps), ranges_string, len(all_ranges), weighting, chunk_idx))
            chunk_idx += 1
            all_cps = []
            all_ranges = []
            line_idx = 0
        line = line[:-1]
        if not line.startswith('#') and len(line) != 0:
            comment_start = line.find('#')
            comment = ''
            if comment_start != -1:
                comment = line[comment_start + 1:].strip()
                line = line[:comment_start]
            if 'surrogate' in comment:
                continue
            if 'noncharacter' in comment:
                continue
            cps = line.split(';')[0].split(' ')
            first = len(all_cps)
            all_cps += cps
            last = len(all_cps)
            all_ranges.append((first, last))
            line_idx += 1

    if line_idx != 0:
        cps_string = ', '.join(map(lambda x: '0x' + x, all_cps))
        ranges_string = ', '.join(map(lambda x: '{{{}, {}}}'.format(x[0], x[1]), all_ranges))
        cpp_file = open('relative_collation_test_{0}_{1}.cpp'.format(weighting, chunk_idx), 'w')
        cpp_file.write(relative_collation_tests_form.format(cps_string, len(all_cps), ranges_string, len(all_ranges), weighting, chunk_idx))

import sys
if '--perf' in sys.argv:
    cet = get_frac_uca_cet('FractionalUCA.txt')
    generate_lookup_perf_test(cet)
    generate_collation_perf_test(cet)
    exit(0)

data_file_test_form = '''\
// Copyright (C) 2020 T. Zachary Laine
//
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

// Warning! This file is autogenerated.
#include <boost/text/collation_table.hpp>
#include <boost/text/collate.hpp>
#include <boost/text/normalize_string.hpp>
#include <boost/text/data/all.hpp>

#include <gtest/gtest.h>


using namespace boost;

auto const error = [](std::string const & s) {{ std::cout << s; }};
auto const warning = [](std::string const & s) {{}};

{0}
'''

single_data_file_test_form = '''\

//{0}
TEST(collation_and_tailoring, data_file_test_{1})
{{
    auto const table = {2};

{3}
}}
'''

def make_string(s, rules, view):
    type_ = 'std::string'
    if view:
        type_ += '_view'
    if rules:
        retval = '{}(u8R"({})"'.format(type_, s)
    else:
        retval = '{}(u8"{}"'.format(type_, s)
    if view and 'x00' in s:
        if rules:
            retval += ', {}'.format(len(s))
        else:
            retval += ', {}'.format(len(eval("'" + s + "'")))
    retval += ')'
    return retval

def comparison_tests(compares, test_strength):
    retval = ''
    prev = ''
    for c in compares:
        (compare, curr) = filter(lambda x: len(x) != 0, c[0].split(' '))
        if compare == '=':
            strength = test_strength
            result = 0
        else:
            result = -1
            if compare == '<':
                strength = test_strength
            if compare == '<i':
                strength = 'identical'
            if compare == '<4':
                strength = 'quaternary'
            if compare == '<3':
                strength = 'tertiary'
            if compare == '<2':
                strength = 'secondary'
            if compare == '<1':
                strength = 'primary'
        retval += '''\

    {{
        // {0} # {1}

        std::string a = {2};
        std::string b = {3};
        normalize<boost::text::nf::fcc>(a);
        normalize<boost::text::nf::fcc>(b);

        EXPECT_EQ(text::collate(
            text::utf32_range(a),
            text::utf32_range(b),
            table,
            text::collation_strength::{4}),
            {5});
    }}
'''.format(c[0], c[1], make_string(prev, False, False), make_string(curr, False, False), strength, result)
        prev = curr
    return retval

# Expects a file like the collationtest.txt file in ICU.
def generate_datafile_collation_tests(lines):
    test_lines = ''
    test_idx = 0
    line_idx = 0
    test_comment = ''
    rules = ''
    strength = 'tertiary'
    compares = []
    skip_rules = False
    while line_idx < len(lines):
        line = lines[line_idx].strip()
        if line.startswith('@'):
            strength = 'tertiary'

        if line.startswith('#') or len(line) == 0:
            line_idx += 1
            continue
        elif line.startswith('** test'):
            test_comment = line[len('** test:'):]
            line_idx += 1
        elif line.startswith('@ root'):
            skip_rules = False
            rules = 'default'
            line_idx += 1
        elif line.startswith('@ rules'):
            skip_rules = False
            rules = ''
            line_idx += 1
            line = lines[line_idx].strip()
            while not line.startswith('*') and not line.startswith('%'):
                comment_start = line.find('#')
                comment = ''
                if comment_start != -1:
                    comment = line[comment_start + 1:].strip()
                    line = line[:comment_start].strip()
                rules += line + ' '
                line_idx += 1
                if len(lines) <= line_idx:
                    break
                line = lines[line_idx].strip()
            if 'import' in rules or '[reorder others]' in rules or \
                'maxVariable' in rules or '@' in rules:
                rules = ''
                skip_rules = True
        elif line.startswith('@ locale'):
            line_idx += 1
            skip_rules = True
        elif line.startswith('% '):
            if line[2:].startswith('reorder '):
                if 'default' in line:
                    rules = 'default'
                else:
                    if rules == 'default':
                        rules = ''
                    rules += '[' + line[2:] + ']'
            else:
                if rules == 'default':
                    rules = ''
                if 'strength=primary' in line:
                    rules += '[strength 1]'
                    strength = 'primary'
                elif 'strength=secondary' in line:
                    rules += '[strength 2]'
                    strength = 'secondary'
                elif 'strength=tertiary' in line:
                    rules += '[strength 3]'
                    strength = 'tertiary'
                elif 'strength=quaternary' in line:
                    rules += '[strength 4]'
                    strength = 'quaternary'
                elif 'strength=identical' in line:
                    rules += '[strength I]'
                    strength = 'identical'
                elif 'backwards=on' in line:
                    rules += '[backwards 2]'
                else:
                    rules += '[' + line[2:].replace('=', ' ') + ']'
            line_idx += 1
            if 'numeric' in rules or 'maxVariable' in rules or 'import' in rules:
                rules = ''
                skip_rules = True
        elif line.startswith('* compare'):
            line_idx += 1
            compares = []
            line = lines[line_idx].strip()
            while not line.startswith('*') and not line.startswith('@') and not line.startswith('%'):
                comment_start = line.find('#')
                comment = ''
                if comment_start != -1:
                    comment = line[comment_start + 1:].strip()
                    line = line[:comment_start].strip()
                if line != '' and '\\ud800' not in line.lower() \
                  and '\\ufff' not in line.lower() \
                  and '\\udb' not in line.lower() and '\\udc' not in line.lower() \
                  and '\\udf' not in line.lower():
                    compares.append((line, comment))
                line_idx += 1
                if len(lines) <= line_idx:
                    break
                line = lines[line_idx].strip()
            table = 'text::default_collation_table()'
            if len(compares) == 0:
                test_idx += 1
            if skip_rules or len(compares) == 0 or '\\ud800' in rules.lower() \
              or '\\udb' in rules.lower() or '\\udc' in rules.lower() \
              or '\\udf' in rules.lower():
              continue
            if rules != 'default' and rules != '':
                table = '''text::tailored_collation_table(
        {0},
        "rules", error, warning)'''.format(
            make_string(rules.replace(';', '<<').replace(',', '<<<'), True, True)
        )
            test_lines += single_data_file_test_form.format(
                test_comment, test_idx, table, comparison_tests(compares, strength)
            )
            test_idx += 1
        else:
            line_idx += 1
    f = open('tailoring_data.cpp', 'w')
    f.write(data_file_test_form.format(test_lines))

if '--file' in sys.argv:
    file_idx = sys.argv.index('--file') + 1
    lines = open(sys.argv[file_idx], 'r').readlines()
    generate_datafile_collation_tests(lines)
    exit(0)

generate_relative_collation_tests('CollationTest_CLDR_NON_IGNORABLE.txt', 'non_ignorable')
generate_relative_collation_tests('CollationTest_CLDR_SHIFTED.txt', 'shifted')
