tokenize method Null safety

  1. @override
Future<List<Token>> tokenize(
  1. SourceText text,
  2. [Zone? zone]
)
override

Extracts tokens from source for use in full-text search queries and indexes.

Optional parameter zone is the name of the zone in a document in which the term is located.

Returns a List<Token>.

Implementation

@override
Future<List<Token>> tokenize(SourceText text, [Zone? zone]) async {
  int position = 0;
// perform the first punctuation and white-space split
  final terms = configuration.termSplitter(text.trim());
  // initialize the tokens collection (return value)
  final tokens = <Token>[];

  // iterate through the terms
  await Future.forEach(terms, (String term) async {
    // remove white-space at start and end of term
    term = term.trim();
    // only tokenize non-empty strings.
    if (term.isNotEmpty) {
      // apply the termFilter if it is not null
      final splitTerms = await configuration.termFilter(term);
      for (var splitTerm in splitTerms) {
        if (splitTerm.isNotEmpty) {
          tokens.add(Token(splitTerm, position, zone));
        }
      }
    }
    // increment the index
    position = position++;
  });
  // apply the tokenFilter if it is not null and return the tokens collection
  return tokenFilter != null ? await tokenFilter!(tokens) : tokens;
}