# Source code for nlp_architect.models.absa.train.acquire_terms

    # ******************************************************************************
    # Copyright 2017-2018 Intel Corporation
    #
    # Licensed under the Apache License, Version 2.0 (the "License");
    # you may not use this file except in compliance with the License.
    # You may obtain a copy of the License at
    #
    #     http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    # ******************************************************************************
    import copy
    import re
    import sys
    from os import PathLike

    from tqdm import tqdm

    from nlp_architect.models.absa import TRAIN_LEXICONS, LEXICONS_OUT
    from nlp_architect.models.absa import GENERIC_OP_LEX
    from nlp_architect.models.absa.inference.data_types import Polarity
    from nlp_architect.models.absa.train.data_types import (
        AspectTerm,
        DepRelation,
        DepRelationTerm,
        LoadOpinionStopLists,
        LoadAspectStopLists,
        OpinionTerm,
        QualifiedTerm,
    )
    from nlp_architect.models.absa.train.rules import rule_1, rule_2, rule_3, rule_4, rule_5, rule_6
    from nlp_architect.models.absa.utils import (
        _load_parsed_docs_from_dir,
        _write_final_opinion_lex,
        _load_lex_as_list_from_csv,
        read_generic_lex_from_file,
    )

    class AcquireTerms(object):
        """
        Lexicon acquisition. produce opinion lexicon and an aspect lexicon based
        on input dataset.

        Attributes:
            opinion_candidate_list_curr_iter (dict): candidate opinion terms in the current iteration
            opinion_candidate_list_prev_iter (dict): opinion candidates list of previous iteration
            opinion_candidate_list (dict): opinion terms learned across all iterations
            opinion_candidates_list_final (list): final opinion candidates list
            opinion_candidate_list_raw (dict): all instances of candidate opinion terms
                                               across all iterations
            aspect_candidate_list_curr_iter (dict): candidate terms in the current iteration
            aspects_candidate_list_prev_iter(list): Aspect candidates list of previous iteration
            aspect_candidate_list (list):  aspect terms learned across all iterations
            aspect_candidates_list_final (list): final aspect candidates list
            aspect_candidate_list_raw (dict): all instances of candidate aspect terms
                                              across all iterations
            """

        generic_opinion_lex_path = GENERIC_OP_LEX
        acquired_opinion_terms_path = LEXICONS_OUT / "generated_opinion_lex.csv"
        acquired_aspect_terms_path = LEXICONS_OUT / "generated_aspect_lex.csv"

        GENERIC_OPINION_LEX = _load_lex_as_list_from_csv(GENERIC_OP_LEX)
        GENERAL_ADJECTIVES_LEX = _load_lex_as_list_from_csv(TRAIN_LEXICONS / "GeneralAdjectivesLex.csv")
        GENERIC_QUANTIFIERS_LEX = _load_lex_as_list_from_csv(
            TRAIN_LEXICONS / "GenericQuantifiersLex.csv"
        )
        GEOGRAPHICAL_ADJECTIVES_LEX = _load_lex_as_list_from_csv(
            TRAIN_LEXICONS / "GeographicalAdjectivesLex.csv"
        )
        INTENSIFIERS_LEX = _load_lex_as_list_from_csv(TRAIN_LEXICONS / "IntensifiersLex.csv")
        TIME_ADJECTIVE_LEX = _load_lex_as_list_from_csv(TRAIN_LEXICONS / "TimeAdjectiveLex.csv")
        ORDINAL_NUMBERS_LEX = _load_lex_as_list_from_csv(TRAIN_LEXICONS / "OrdinalNumbersLex.csv")
        PREPOSITIONS_LEX = _load_lex_as_list_from_csv(TRAIN_LEXICONS / "PrepositionsLex.csv")
        PRONOUNS_LEX = _load_lex_as_list_from_csv(TRAIN_LEXICONS / "PronounsLex.csv")
        COLORS_LEX = _load_lex_as_list_from_csv(TRAIN_LEXICONS / "ColorsLex.csv")
        DETERMINERS_LEX = _load_lex_as_list_from_csv(TRAIN_LEXICONS / "DeterminersLex.csv")
        NEGATION_LEX = _load_lex_as_list_from_csv(TRAIN_LEXICONS / "NegationLex.csv")
        AUXILIARIES_LEX = _load_lex_as_list_from_csv(TRAIN_LEXICONS / "AuxiliariesLex.csv")

        OPINION_STOP_LIST = LoadOpinionStopLists(
            DETERMINERS_LEX,
            GENERAL_ADJECTIVES_LEX,
            GENERIC_QUANTIFIERS_LEX,
            GEOGRAPHICAL_ADJECTIVES_LEX,
            INTENSIFIERS_LEX,
            TIME_ADJECTIVE_LEX,
            ORDINAL_NUMBERS_LEX,
            PREPOSITIONS_LEX,
            COLORS_LEX,
            NEGATION_LEX,
        )

        ASPECT_STOP_LIST = LoadAspectStopLists(
            GENERIC_OPINION_LEX,
            DETERMINERS_LEX,
            GENERAL_ADJECTIVES_LEX,
            GENERIC_QUANTIFIERS_LEX,
            GEOGRAPHICAL_ADJECTIVES_LEX,
            INTENSIFIERS_LEX,
            TIME_ADJECTIVE_LEX,
            ORDINAL_NUMBERS_LEX,
            PREPOSITIONS_LEX,
            PRONOUNS_LEX,
            COLORS_LEX,
            NEGATION_LEX,
            AUXILIARIES_LEX,
        )

        FILTER_PATTERNS = [re.compile(r".*\d+.*")]
        FLOAT_FORMAT = "{0:.3g}"
        # maximum number of iterations
        NUM_OF_SENTENCES_PER_OPINION_AND_ASPECT_TERM_INC = 35000

        def __init__(self, asp_thresh=3, op_thresh=2, max_iter=1):
            self.opinion_candidate_list_prev_iter = read_generic_lex_from_file(
                AcquireTerms.generic_opinion_lex_path
            )
            self.generic_sent_dict = copy.deepcopy(self.opinion_candidate_list_prev_iter)
            self.opinion_candidate_list = {}
            self.opinion_candidate_list_raw = {}
            self.opinion_candidate_list_curr_iter = {}
            self.opinion_candidates_list_final = []
            self.aspect_candidate_list_raw = {}
            self.aspect_candidate_list = list()
            self.aspect_candidate_list_curr_iter = {}
            self.aspect_candidates_list_final = []
            self.init_aspect_dict = list()
            self.aspects_candidate_list_prev_iter = list()
            self.min_freq_aspect_candidate = asp_thresh
            self.min_freq_opinion_candidate = op_thresh
            self.max_num_of_iterations = max_iter

        def extract_terms_from_doc(self, parsed_doc):
            """Extract candidate terms for sentences in parsed document.

            Args:
                parsed_doc (ParsedDocument): Input parsed document.
            """
            for text, parsed_sent in parsed_doc.sent_iter():
                relations = _get_rel_list(parsed_sent)

                for rel_entry in relations:
                    if rel_entry.rel != "root":
                        gov_seen = self.opinion_candidate_list_prev_iter.get(rel_entry.gov.text)
                        dep_seen = self.opinion_candidate_list_prev_iter.get(rel_entry.dep.text)
                        opinions = []
                        aspects = []

                        # =========================== acquisition rules ==============================

                        if bool(gov_seen) ^ bool(dep_seen):
                            opinions.append(rule_1(rel_entry, gov_seen, dep_seen, text))

                        if not gov_seen and dep_seen:
                            opinions.append(rule_2(rel_entry, relations, dep_seen, text))

                            aspects.append(rule_3(rel_entry, relations, text))

                            aspects.append(rule_4(rel_entry, relations, text))

                        if (
                            self.aspects_candidate_list_prev_iter
                            and AspectTerm.from_token(rel_entry.gov)
                            in self.aspects_candidate_list_prev_iter
                            and AspectTerm.from_token(rel_entry.dep)
                            not in self.aspects_candidate_list_prev_iter
                        ):
                            opinions.append(rule_5(rel_entry, text))
                            aspects.append(rule_6(rel_entry, relations, text))

                        self._add_opinion_term(opinions)
                        self._add_aspect_term(aspects)

        def extract_opinion_and_aspect_terms(self, parsed_document_iter, num_of_docs):
            """Extract candidate terms from parsed document iterator.

            Args:
                parsed_document_iter (Iterator): Parsed document iterator.
                num_of_docs (int): number of documents on iterator.
            """

            for parsed_document in tqdm(parsed_document_iter, total=num_of_docs, file=sys.stdout):
                self.extract_terms_from_doc(parsed_document)

        def _is_valid_term(self, cand_term):
            """Validates a candidate term.

            Args:
                cand_term (CandidateTerm): candidate terms list.
            """
            term = str(cand_term)
            for pattern in self.FILTER_PATTERNS:
                if pattern.match(term):
                    return False
            if self.OPINION_STOP_LIST.is_in_stop_list(term):
                return False
            if term.lower() != term and term.upper() != term:
                return False
            return True

        def _add_aspect_term(self, terms):
            """
            add new aspect term to table.
            Args:
                terms (list of CandidateTerm): candidate terms list
            """
            for term in terms:
                if term:
                    term_entry = AspectTerm(term.term, term.pos, term.lemma)
                    if (
                        term_entry not in self.init_aspect_dict
                        and term_entry not in self.aspect_candidate_list
                        and not self.ASPECT_STOP_LIST.is_in_stop_list(term.term[0])
                        and len(term.term[0]) > 1
                    ):
                        _insert_new_term_to_table(term, self.aspect_candidate_list_curr_iter)

            return True

        def _add_opinion_term(self, terms):
            """
            Add new opinion term to table
            Args:
                terms (list of CandidateTerm): candidate term
            """
            for term in terms:
                if term and self._is_valid_term(term):
                    if str(term.term[0]) not in self.generic_sent_dict.keys():
                        if str(term.term[0]) not in self.opinion_candidate_list:
                            if len(str(term.term[0])) > 1:
                                if any(c.isalnum() for c in str(term.term[0])):
                                    _insert_new_term_to_table(
                                        term, self.opinion_candidate_list_curr_iter
                                    )

        def _insert_new_terms_to_tables(self):
            """
            Insert new terms to tables
            clear candidates lists from previous iteration

            """
            self.opinion_candidate_list_prev_iter = {}
            self.opinion_candidate_list_raw = _merge_tables(
                self.opinion_candidate_list_raw, self.opinion_candidate_list_curr_iter
            )
            for cand_term_list in self.opinion_candidate_list_curr_iter.values():
                if len(cand_term_list) >= self.min_freq_opinion_candidate:
                    new_opinion_term = _set_opinion_term_polarity(cand_term_list)
                    self.opinion_candidate_list_prev_iter[str(new_opinion_term)] = new_opinion_term
            self.opinion_candidate_list_curr_iter = {}
            self.opinion_candidate_list = {
                **self.opinion_candidate_list,
                **self.opinion_candidate_list_prev_iter,
            }
            self.aspects_candidate_list_prev_iter = list()
            self.aspect_candidate_list_raw = _merge_tables(
                self.aspect_candidate_list_raw, self.aspect_candidate_list_curr_iter
            )
            for extracted_aspect_list in self.aspect_candidate_list_curr_iter.values():
                if len(extracted_aspect_list) >= self.min_freq_aspect_candidate:
                    first = extracted_aspect_list[0]
                    new_aspect_entry = AspectTerm(first.term, first.pos, first.lemma)
                    if new_aspect_entry not in self.aspects_candidate_list_prev_iter:
                        self.aspects_candidate_list_prev_iter.append(new_aspect_entry)
            self.aspect_candidate_list_curr_iter = {}
            self.aspect_candidate_list = (
                self.aspect_candidate_list + self.aspects_candidate_list_prev_iter
            )

        def _write_candidate_opinion_lex(self):
            """
            write generated lexicons to csv files
            """
            LEXICONS_OUT.mkdir(parents=True, exist_ok=True)

            _write_final_opinion_lex(
                self.opinion_candidates_list_final, self.acquired_opinion_terms_path
            )

        def acquire_lexicons(self, parsed_dir: str or PathLike):
            """Acquire new opinion and aspect lexicons.

            Args:
                parsed_dir (PathLike): Path to parsed documents folder.
            """
            parsed_docs = _load_parsed_docs_from_dir(parsed_dir)
            dataset_sentence_len = 0
            for parsed_doc in parsed_docs.values():
                dataset_sentence_len += len(parsed_doc.sentences)

            add_to_thresholds = int(
                dataset_sentence_len / self.NUM_OF_SENTENCES_PER_OPINION_AND_ASPECT_TERM_INC
            )
            self.min_freq_opinion_candidate += add_to_thresholds
            self.min_freq_aspect_candidate += add_to_thresholds

            for iteration_num in range(self.max_num_of_iterations):
                if (
                    len(self.opinion_candidate_list_prev_iter) == 0
                    and len(self.aspects_candidate_list_prev_iter) == 0
                ):
                    break

                print("\n#Iteration: {}".format(iteration_num + 1))

                self.extract_opinion_and_aspect_terms(iter(parsed_docs.values()), len(parsed_docs))

                self._insert_new_terms_to_tables()

            self.opinion_candidates_list_final = generate_final_opinion_candidates_list(
                self.opinion_candidate_list_raw,
                self.opinion_candidates_list_final,
                self.min_freq_opinion_candidate,
            )
            self.aspect_candidates_list_final = _generate_final_aspect_candidates_list(
                self.aspect_candidate_list_raw,
                self.aspect_candidates_list_final,
                self.min_freq_aspect_candidate,
            )

            self._write_candidate_opinion_lex()

            aspect_dict = _add_lemmas_aspect_lex(self.aspect_candidates_list_final)

            return aspect_dict

    def _add_lemmas_aspect_lex(aspect_candidates_list_final):

        aspect_dict = {}
        for cand_term in aspect_candidates_list_final:
            lemma = ""
            if cand_term.term[0] != cand_term.lemma[0]:
                lemma = cand_term.lemma[0]
            aspect_dict[cand_term.term[0]] = lemma

        # unify aspect with aspect lemmas
        lemma_to_erase = []
        for _, lemma in aspect_dict.items():
            if lemma != "" and lemma in aspect_dict:
                lemma_to_erase.append(lemma)

        # delete all duplicates (aspects that are lemmas of other aspects)
        for lemma in lemma_to_erase:
            if lemma in aspect_dict:
                del aspect_dict[lemma]
        return aspect_dict

    def _get_rel_list(parsed_sentence):
        res = []
        gen_toks = []
        for tok in parsed_sentence:
            gen_toks.append(
                DepRelationTerm(tok["text"], tok["lemma"], tok["pos"], tok["ner"], tok["start"])
            )

        for gen_tok, tok in zip(gen_toks, parsed_sentence):
            gov_idx = tok["gov"]
            if gov_idx != -1:
                res.append(DepRelation(gen_toks[gov_idx], gen_tok, tok["rel"]))
        return res

    def _merge_tables(d1, d2):
        """
        Merge dictionaries
        Args:
            d1 (dict): first dict to merge
            d2 (dict): second dict to merge
        """
        for key, l in d2.items():
            if key in d1:
                for item in l:
                    if item not in d1[key]:
                        d1[key].append(item)
            else:
                d1[key] = l
        return d1

    def _insert_new_term_to_table(term, curr_table):
        """
        Insert term to table of lists.
        Args:
            term (term): term to be inserted
            curr_table (dict): input table
        """
        table_key_word = str(term)
        if table_key_word:
            if table_key_word in curr_table and term not in curr_table[table_key_word]:
                curr_table[table_key_word].append(term)
            else:
                curr_table[table_key_word] = [term]

    def _set_opinion_term_polarity(terms_list):
        """Set opinion term polarity.

        Args:
            terms_list (list): list of opinion terms
        """
        first = terms_list[0]
        new_term = first.term

        positive_pol = 0
        negative_pol = 0
        pol = None
        for term in terms_list:
            try:
                pol = term.term_polarity
            except Exception as e:
                print("extracted_term missing term_polarity: " + str(e))
            if pol is not None:
                if pol == Polarity.POS:
                    positive_pol = positive_pol + 1
                if pol == Polarity.NEG:
                    negative_pol = negative_pol + 1
        new_term_polarity = Polarity.UNK
        if positive_pol >= negative_pol and positive_pol > 0:
            new_term_polarity = Polarity.POS
        elif negative_pol >= positive_pol and negative_pol > 0:
            new_term_polarity = Polarity.NEG

        return OpinionTerm(new_term, new_term_polarity)

    def _generate_final_aspect_candidates_list(
        aspect_candidate_list_raw, final_aspect_candidates_list, frequency_threshold
    ):
        """
        generate final aspect candidates list from map
        Args:
            aspect_candidate_list_raw (dict): key = term, value =
            lists of candidate terms.
            final_aspect_candidates_list (list): list of final aspect candidates
            frequency_threshold (int): minimum freq. for qualifying term
        """
        term_polarity = Polarity.UNK
        for extracted_term_list in aspect_candidate_list_raw.values():
            if len(extracted_term_list) >= frequency_threshold:
                term = extracted_term_list[0]
                qualified_term = QualifiedTerm(
                    term.term, term.lemma, term.pos, len(extracted_term_list), term_polarity
                )
                final_aspect_candidates_list.append(qualified_term)

        return final_aspect_candidates_list

    def generate_final_opinion_candidates_list(
        opinion_candidate_list_raw, final_opinion_candidates_list, frequency_threshold
    ):
        """
        generate final opinion candidates list from raw opinion candidate list
        Args:
            opinion_candidate_list_raw (dict): key = term, value =
            lists of extracted terms.
            final_opinion_candidates_list (list): list of final opinion candidates
            frequency_threshold (int): minimum freq. for qualifying term
        """
        for candidate_list in opinion_candidate_list_raw.values():
            positive_pol = 0
            negative_pol = 0
            if len(candidate_list) >= frequency_threshold:
                for candidate in candidate_list:
                    pol = candidate.term_polarity
                    if pol is not None:
                        if pol == Polarity.POS:
                            positive_pol = positive_pol + 1
                        if pol == Polarity.NEG:
                            negative_pol = negative_pol + 1

                term_polarity = Polarity.UNK
                if positive_pol > negative_pol and positive_pol > 0:
                    term_polarity = Polarity.POS
                elif negative_pol >= positive_pol and negative_pol > 0:
                    term_polarity = Polarity.NEG

                term = candidate_list[0]

                qualified_term = QualifiedTerm(
                    term.term, term.term, term.pos, len(candidate_list), term_polarity
                )
                final_opinion_candidates_list.append(qualified_term)

        return final_opinion_candidates_list