% kate: indent-width 2; word-wrap-column 74; syntax latex

%!TEX program = XeLaTeX


% sudo apt install latex-cjk-all


%  \documentclass[nojss]{jss}
\documentclass[nojss]{jss}
\hypersetup{xetex,colorlinks=true}
\usepackage{xltxtra}

\usepackage{thumbpdf}
\usepackage{lmodern}
\usepackage{xcolor}
% \usepackage{framed}
% \usepackage[utf8]{inputenc}
% \usepackage[T1]{fontenc}
% \usepackage{CJKutf8}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{booktabs}
\usepackage{verbatim}
\usepackage{tabularx}
% \usepackage[UTF8]{ctex}
\usepackage{xeCJK}
\usepackage{fontspec}
\usepackage{wasysym}
\setCJKmainfont{Noto Sans Mono CJK SC}
\setCJKsansfont{Noto Sans Mono CJK SC}
\setCJKmonofont{Noto Sans Mono CJK SC}
\usepackage{tipa}
% \usepackage{lmodern}
% \setmonofont{Latin Modern Mono}
% \setmonofont{Ubuntu Mono}%[Scale=0.85]
% \usepackage{alphabeta}




\author{Marek Gagolewski\\
Deakin University, Geelong, VIC 3220, Australia\\
Systems Research Institute, Polish Academy of Sciences
}
\Plainauthor{Marek Gagolewski}

\title{\pkg{stringi}: Fast and Portable\\Character String
Processing in \proglang{R}}
\Plaintitle{stringi: Fast and Portable Character String Processing in R}
\Shorttitle{\pkg{stringi}: Fast and Portable Character String Processing
in \proglang{R}}




\Abstract{
Effective processing of character strings is required at various
stages of data analysis pipelines:
from data cleansing and preparation, through
information extraction, to report generation.
Pattern searching, string collation and sorting, normalisation,
transliteration, and formatting are ubiquitous in
text mining, natural language processing, and bioinformatics.
This paper discusses and demonstrates how and why \pkg{stringi},
a mature \proglang{R} package for fast and portable handling of string data
based on \pkg{International Components for Unicode},
should be included in each statistician's or data scientist's
repertoire to complement their  numerical computing
and data wrangling skills.
}

\Keywords{\pkg{stringi}, character strings, text, \pkg{ICU}, Unicode,
regular expressions, data cleansing, natural language processing,
\proglang{R}}
\Plainkeywords{stringi, character strings, text, ICU, Unicode, regular
expressions, data cleansing, natural language processing, R}

\Address{
  \textbf{Marek Gagolewski}\\
  School of Information Technology\\
  Deakin University\\
  Geelong, VIC 3220, Australia\\
  \emph{and}\\
  Systems Research Institute\\
  Polish Academy of Sciences\\
  ul.~Newelska 6, 01-447 Warsaw, Poland\\
  E-mail: \email{m.gagolewski@deakin.edu.au}\\
  URL: \url{https://www.gagolewski.com/}
}

\begin{document}

{\color{blue}
This is an older pre-print version of the paper on \pkg{stringi}.

Please cite it as:
Gagolewski M (2022).
\pkg{stringi}: Fast and Portable Character String Processing in \proglang{R}.
\textit{Journal of Statistical Software} 103(2):1--59, 2022.
DOI \url{https://dx.doi.org/10.18637/jss.v103.i02}.

The most recent, Web browser-friendly version thereof
is available at \url{https://stringi.gagolewski.com}.
}


<<echo=FALSE, results="hide", message=FALSE>>=
#options(prompt="R> ", continue="+  ")
library("knitr")
#knitr::render_sweave()
#knitr::knitr_hooks$set(chunk=knitr::render_sweave())

opts_chunk$set(
#  fig.height=3.5,
#  fig.width=6,
#  dev=c("CairoPDF", "CairoSVG"),
#  out.width=NULL,
#  dpi=300,
#  dev.args=list(pointsize=11),
#  fig.show="hold",
#  fig.lp='fig:',
  tidy=FALSE,
  error=FALSE,
  autodep=TRUE,
  cache=TRUE
)

library("stringi")
stri_locale_set("en_AU")
library("stringr")
set.seed(123)

options(width=77)
options(useFancyQuotes=FALSE)
options(encoding="UTF-8")
options(digits=5)
options(max.print=99)
options(scipen=10)
options(showWarnCalls=FALSE)
#options(stringsAsFactors=FALSE) # default in R 4.0
##########################################################################
@




\section{Introduction}\label{Sec:intro}

Stringology \citep{stringology} deals with algorithms and data structures
for character string processing \citep{speechlangproc,szpankowski}.
From the perspective of applied statistics and data science,
it is worth stressing that many interesting data sets
first come in unstructured or contaminated textual forms,
for instance when they have been fetched from different APIs
(application programming interface) or gathered
by means of web scraping techniques.

Diverse data cleansing and preparation operations
(\citealp{datacleaning,cleanr}; see also
Section~\ref{Sec:examples} below for a real-world example)
need to be applied before an analyst can begin to enjoy
an orderly and meaningful data frame, matrix, or spreadsheet being finally
at their disposal.
Activities related to information retrieval, computer vision,
bioinformatics, natural language processing, or even musicology
can also benefit from including them
in data processing pipelines \citep{speechlangproc,genome}.


Although statisticians and data analysts are usually
very proficient in numerical computing and data wrangling,
the awareness of how crucial text operations are in the generic
data-oriented skill-set is yet to reach a more operational
level. This paper aims to fill this gap.

\bigskip
Most statistical computing ecosystems provide only a basic set of text
operations. In particular, base \proglang{R} \citep{Rproject:home}
is mostly restricted to pattern matching, string concatenation,
substring extraction, trimming, padding, wrapping,
simple character case conversion, and string collation,
see \citep[Chapter~8]{Chambers2008:SoftDA} and Table~\ref{Tab:oldstringr} below.
The \pkg{stringr} package \citep{Wickham2010:stringr},
first released in November 2009, implemented an alternative, ``tidy''
API for text data processing (cleaned-up function names,
more beginner-friendly outputs, etc.;
the list of 21 functions that were available in \pkg{stringr} at that
time is given in Table~\ref{Tab:oldstringr}).
The early \pkg{stringr} featured a few wrappers around
a subset of its base \proglang{R} counterparts. The latter,
however -- to this day -- not only is of limited scope,
but also suffers from a number of portability issues; it may happen that
the same code can yield different results on different operating systems;
see Section~\ref{Sec:design} for some examples.



In order to significantly broaden the array of string processing
operations and assure that they are portable, in 2013 the current
author developed the open source \pkg{stringi} package
(pronounced ``stringy'', IPA \textipa{[{stringi}]}).
Its API was compatible with that of early \pkg{stringr}'s,
which some users found convenient. However, for the processing of text
in different locales, which are plentiful, \pkg{stringi} relies
on \pkg{ICU} -- \pkg{International Components for Unicode}
(see \url{https://icu.unicode.org/}) -- a mature library that
fully conforms with the Unicode standard and which provides globalisation
support for a broad range of other software applications as well, from
web browsers to database systems. Services not covered by \pkg{ICU}
were implemented from scratch to guarantee that they are as efficient
as possible.

Over the years, \pkg{stringi} confirmed itself as robust,
production-quality software; for many years now it has been one of the
most often downloaded \proglang{R} extensions. Interestingly, in 2015 the
aforementioned \pkg{stringr} package has been rewritten as a
set of wrappers around some of the \pkg{stringi} functions instead of
the base \proglang{R} ones.
In Section~14.7 of \textit{\proglang{R} for Data Science}
\citep{GrolemundWickham2017:rdatascience}
we read: \textit{\pkg{stringr} is useful when you're learning because
it exposes a minimal set of functions, which have been carefully picked
to handle the most common string manipulation functions. \pkg{stringi},
on the other hand, is designed to be comprehensive. It contains almost
every function you might ever need: \pkg{stringi} has 250 functions to
\pkg{stringr}'s 49}.
Also, it is worth noting that the recently-introduced \pkg{stringx}
package \citep{stringx} supplies a \pkg{stringi}-based set of portable
and efficient replacements for and enhancements of the base \proglang{R}
functions.



\bigskip
This paper describes the most noteworthy facilities provided by
\pkg{stringi} that statisticians and data analysts may find
useful in their daily activities. We demonstrate how important it is
for a modern data scientist to be aware of the challenges
of natural language processing in the internet era:
how to force \code{"groß"} compare equal to \code{"GROSS"},
count the number of occurrences of \code{"AGA"} within
\code{"ACTGAGAGACGGGTTAGAGACT"},
make \code{"a13"} ordered before \code{"a100"}, or
convert between \code{"GRINNING FACE"} and \code{"\smiley"} forth and back.
Such operations are performed by the very \pkg{ICU} itself;
we therefore believe that what follows may be of interest
to data-oriented practitioners employing \proglang{Python},
\proglang{Perl}, \proglang{Julia}, \proglang{PHP}, etc.,
as \pkg{ICU} has bindings for many other languages.


Here is the outline of the paper:
\begin{itemize}
\item Section~\ref{Sec:examples} illustrates the importance of string
processing in an example data preparation activity.

\item General package design principles are outlined in
Section~\ref{Sec:design}, including the use cases of deep vectorisation,
the concepts of data flow,
and the main deviations from base \proglang{R}
(also with regards to portability and speed).

\item Basic string operations, such as computing length and width
of strings, string concatenation, extracting and replacing substrings,
are discussed in Section~\ref{Sec:basic}.

\item Section~\ref{Sec:fixed} discusses searching for fixed substrings:
counting the number of matches, locating their positions,
replacing them with other data, and splitting strings into tokens.

\item Section~\ref{Sec:regex} details \pkg{ICU} regular
expressions, which are a powerful tool for matching patterns defined in
a more abstract way, e.g., extracting numbers from text so that they can
be processed quantitatively, identifying hyperlinks, etc.
We show where \pkg{ICU} is different from other libraries like \pkg{PCRE};
in particular that it enables portable, Unicode-correct look-ups,
for instance, involving sequences of emojis or mathematical symbols.

\item Section~\ref{Sec:collator} deals with the locale-aware \pkg{ICU}
Collator, which is  suitable for natural language processing activities;
this is where we demonstrate that text processing in different languages
or regions is governed by quite diverse rules,
deviating significantly from the US-ASCII (``C/POSIX.1'') setting.
The operations discussed therein include testing string equivalence (which can
turn out useful when we scrape data that consist of non-normalised strings,
ignorable punctuation, or accented characters) as well as arranging strings
with regards to different linear orderings.

\item Section~\ref{Sec:other} covers some other useful operations such
as text boundary analysis (for splitting text into words or sentences),
trimming, padding, and other formatting, random string generation,
character transliteration (converting between cases and alphabets,
removing diacritic marks, etc.) as well as date-time formatting and
parsing in any locale (e.g., Japanese dates in a German \proglang{R}).

\item Section~\ref{Sec:io} details on encoding conversion and detection
(which is key when reading or writing text files that are to be communicated
across different systems) as well as Unicode normalisation
(which can be useful for removing formatting
distinctions from text, e.g., superscripts or font variants).

\item Finally, Section~\ref{Sec:conclusions} concludes the paper.
\end{itemize}



This paper is by no means a substitute for the comprehensive yet much
more technical and in-depth reference manual available via a call to
\code{help(package="stringi")}, see also
\url{https://stringi.gagolewski.com/}. Rather, below we explain
the package's key design principles and broadly introduce the ideas and
services that help program, correct, and optimise text processing workflows.

Let us emphasise that all the below-presented illustrations,
i.e., calls to \pkg{stringi} functions on different
example arguments together with the generated outputs, form an integral
part of this manuscript's text. They have been included based on
the author's experience-based belief that each "picture" (that we print
out below using a monospaced font) is worth hundreds of words.











\medskip
All code chunk outputs presented in this paper were obtained in
\proglang{R}~\Sexpr{paste(R.Version()[6:7], collapse=".")}.
The \proglang{R} environment itself and all the packages used herein
are available from CRAN (the Comprehensive \proglang{R} Archive Network)
at \url{https://CRAN.R-project.org/}.
In particular, \code{install.packages("stringi")} can be called to
fetch the object of our focus.
By calling:


<<results="hide">>=
library("stringi")
cat(stri_info(short=TRUE))
@

<<echo=FALSE>>=
##########################################################################
############## TODO: del me - debug only!!! ##########################################################################
stopifnot(!stringi::stri_info()$ICU.system)
stopifnot(packageVersion("stringi") >= package_version("1.7.4"))
cat(stri_wrap(stri_info(short=TRUE), 78), sep="\n")
##########################################################################
@


\noindent
we can load and attach the package's namespace
and display some basic information thereon.
Hence, below we shall be working with
\pkg{stringi} \Sexpr{packageVersion("stringi")}, however, as the package's
API is considered stable, the presented material should be relevant to
any later versions.




\section{Use case: Data preparation}\label{Sec:examples}

Before going into details on the broad array of facilities offered by the
\pkg{stringi} package itself,
let us first demonstrate that string processing is indeed a relevant
part of statistical data analysis workflows.
What follows is a short case study where we prepare a web-scraped
data set for further processing.

Assume we wish to gather and analyse
climate data for major cities around the world based on information
downloaded from Wikipedia.
For each location from a given list of settlements (e.g.,
fetched from one of the pages linked under
\url{https://en.wikipedia.org/wiki/Lists_of_cities}),
we would like to harvest the relevant temperature and precipitation data.
Without harm in generality, let us focus on the city of Melbourne,
VIC, Australia.

The parsing of the city's~Wikipedia page
can be done by means of the functions from the
\pkg{xml2} \citep{xml2}
and \pkg{rvest} \citep{rvest} packages.

<<html-example-1>>=
library("xml2")
library("rvest")
@

First, let us load and parse the \proglang{HTML} file
downloaded on 2020--09--17 (see the accompanying supplementary files):

<<html-example-2>>=
f <- read_html("20200917_wikipedia_melbourne.html")
@

Second, we extract all \code{table} elements and gather them
in a list of \proglang{HTML} nodes, \code{all\_tables}.
We then extract the underlying raw text data and store them in a
character vector named \code{text\_tables}.

<<html-example-3>>=
all_tables <- html_nodes(f, "table")
text_tables <- sapply(all_tables, html_text)
str(text_tables, nchar.max=65, vec.len=5, strict.width="wrap") # preview
@

Most~Wikipedia pages related to particular cities
include a table labelled as ``Climate data''.
We need to pinpoint it amongst all the other tables.
For this, we will rely on \pkg{stringi}'s
\code{stri_detect_fixed()} function that, in the configuration
below, is used to extract the index of the relevant table.

<<html-example-4>>=
library("stringi")
(idx <- which(stri_detect_fixed(text_tables, "climate data",
  case_insensitive=TRUE, max_count=1)))
@

Of course, the detailed description of all the facilities
brought by \pkg{stringi} is covered below.
In the meantime, let us use \pkg{rvest}'s \code{html\_table()}
to convert the above table to a data frame object.


<<html-example-5,results="hide">>=
(x <- as.data.frame(html_table(all_tables[[idx]], fill=TRUE)))
@

<<html-example-6,echo=FALSE>>=
##########################################################################
############## TODO: del me - debug only!!!
##########################################################################
.x <- x[c(1, 2, 3, ncol(x))]
.x[,3] <- "..."
names(.x)[3] <- "..."
print(.x)
##########################################################################
@

It is evident that this object requires some significant cleansing
and transforming before it can be subject to any statistical analyses.
First, for the sake of convenience, let us convert it to a character
matrix so that the processing of all the cells can be vectorised
(a matrix in \proglang{R} is just a single ``long'' vector,
whereas a data frame is a list of many atomic vectors).

<<html-example-7>>=
x <- as.matrix(x)
@

The \code{as.numeric()} function will find the parsing of
the Unicode MINUS SIGN (U+2212, ``−'') difficult,
therefore let us call the transliterator first in order to replace it
(and other potentially problematic characters) with its simpler equivalent:

<<html-example-8>>=
x[, ] <- stri_trans_general(x, "Publishing-Any; Any-ASCII")
@

Note that it is the first row of the matrix that defines the column names.
Moreover, the last row just gives the data source and hence may be removed.

<<html-example-9>>=
dimnames(x) <- list(x[, 1], x[1, ])  # row, column names
x <- x[2:(nrow(x) - 1), 2:ncol(x)]   # skip 1st/last row and 1st column
x[, c(1, ncol(x))]                   # example columns
@

Commas that are used as thousands separators (commas surrounded
by digits) should be dropped:

<<html-example-10>>=
x[, ] <- stri_replace_all_regex(x, "(?<=\\d),(?=\\d)", "")
@

The numbers and alternative units in parentheses are redundant,
therefore these should be taken care of as well:

<<html-example-11>>=
x[, ] <- stri_replace_all_regex(x,
  "(\\d+(?:\\.\\d+)?)\\(\\d+(?:\\.\\d+)?\\)", "$1")
dimnames(x)[[1]] <- stri_replace_all_fixed(dimnames(x)[[1]],
  c(" (°F)", " (inches)"), c("", ""), vectorise_all=FALSE)
@

At last, \code{as.numeric()} can be used to re-interpret all the strings
as numbers:

<<html-example-12>>=
x <- structure(as.numeric(x), dim=dim(x), dimnames=dimnames(x))
x[, c(1, 6, ncol(x))]  # example columns
@

We now have a cleansed matrix at our disposal.
We can, for instance, compute the monthly temperature ranges:

<<html-example-13>>=
x["Record high °C", -ncol(x)] - x["Record low °C", -ncol(x)]
@

\noindent
or the average daily precipitation:

<<html-example-14>>=
sum(x["Average rainfall mm", -ncol(x)]) / 365.25
@

\noindent
and so forth.

For the climate data on other cities, very similar operations
will need to be performed -- the whole process of scraping and cleansing data
can be automated quite easily.
The above functions are not only convenient to use, but also efficient
and portable across different platforms.




\section{General design principles}\label{Sec:design}

The API of the early releases of \pkg{stringi} has been designed so as
to be fairly compatible with that of the 0.6.2 version of the
\pkg{stringr} package
\citep{Wickham2010:stringr} (dated 2012; see Table~\ref{Tab:oldstringr}),
with some fixes in the
consistency of the handling of missing values and zero-length vectors,
amongst others.
However, instead of being merely thin wrappers around base \proglang{R}
functions, which we have identified as not necessarily portable across
platforms and not really suitable for natural language processing tasks,
all the functionality has been implemented from the ground up,
with the use of \pkg{ICU} services wherever applicable. Since the
initial release, an abundance of new features has been added
and the package can now be considered a comprehensive workhorse
for text data processing.
Note that the \pkg{stringi} API is stable.
Future releases are aiming for as much backward compatibility
as possible so that other software projects can safely rely on it.








\subsection{Naming}

Function and argument names use a combination of lowercase letters
and underscores (and no dots). To avoid namespace clashes,
all function names feature the ``\code{stri_}'' prefix.
Names are fairly self-explanatory, e.g.,
\code{stri_locate_first_regex} and \code{stri_locate_all_fixed}
find, respectively, the first match to a regular expression and
all occurrences of a substring as-is.





\subsection{Vectorisation}

Individual character (or code point) strings can be entered using
double quotes or apostrophes:

<<>>=
"spam"  # or 'spam'
@

However, as the  \proglang{R} language does not feature any
classical scalar types, strings are wrapped around atomic vectors
of type ``\code{character}'':

<<>>=
typeof("spam")  # object type; see also is.character() and is.vector()
length("spam")  # how many strings are in this vector?
@

\noindent
Hence, we will be using the terms ``string'' and ``character vector
of length 1'' interchangeably.

Not having a separate scalar type is very convenient; the so-called
\emph{vectorisation} strategy encourages writing code that
processes whole collections of objects, all at once,
regardless of their size.

For instance, given the following character vector:


<<>>=
pythons <- c("Graham Chapman", "John Cleese", "Terry Gilliam",
  "Eric Idle", "Terry Jones", "Michael Palin")
@

\noindent
we can separate the first and the last names from
each other (assuming for simplicity that no middle names are given),
using just a single function call:

<<>>=
(pythons <- stri_split_fixed(pythons, " ", simplify=TRUE))
@

\noindent
Due to {vectorisation}, we can generally
avoid using the \code{for}- and \code{while}-loops
(``for each string in a vector\dots''),
which makes the code much more readable, maintainable, and faster to execute.





\subsection{Acting elementwise with recycling}



Binary and higher-arity operations in \proglang{R}
are oftentimes vectorised with respect to all arguments
(or at least to the crucial, non-optional ones).
As a prototype, let us consider the binary arithmetic,
logical, or comparison operators
(and, to some extent, \code{paste()},
\code{strrep()}, and more generally \code{mapply()}),
for example the multiplication:



<<>>=
c(10, -1) * c(1, 2, 3, 4)  # == c(10, -1, 10, -1) * c(1, 2, 3, 4)
@

\noindent
Calling ``\code{x * y}'' multiplies the corresponding components
of the two vectors elementwisely. As one operand happens to be shorter
than another, the former is recycled as many times as necessary
to match the length of the latter (there would be a warning if partial
recycling occurred). Also, acting on a zero-length
input always yields an empty vector.


All functions in \pkg{stringi} follow this convention
(with some obvious exceptions, such as the \code{collapse}
argument in \code{stri_join()}, \code{locale} in \code{stri_datetime_parse()}, etc.).
In particular, all string search functions are vectorised
with respect to both the  \code{haystack} and the \code{needle} arguments
(and, e.g., the replacement string, if applicable).

Some users, unaware of this rule, might find this behaviour
unintuitive at the beginning and thus miss out on how powerful it is. Therefore, let us enumerate
the most noteworthy scenarios that are possible thanks to the arguments' recycling,
using the call to \code{stri_count_fixed(haystack, needle)}
(which looks for a needle in a haystack)
as an illustration:

\begin{itemize}
\item many strings -- one pattern:

<<>>=
stri_count_fixed(c("abcd", "abcabc", "abdc", "dab", NA), "abc")
@

(there is 1 occurrence of \code{"abc"} in \code{"abcd"},
2 in \code{"abcabc"}, and so forth);

\item one string -- many patterns:

<<>>=
stri_count_fixed("abcdeabc", c("def", "bc", "abc", NA))
@

(\code{"def"} does not occur in \code{"abcdeabc"},
\code{"bc"} can be found therein twice, etc.);

\item each string -- its own corresponding pattern:

<<>>=
stri_count_fixed(c("abca", "def", "ghi"), c("a", "z", "h"))
@

(there are two \code{"a"}s in \code{"abca"},
no \code{"z"} in \code{"def"}, and one \code{"h"} in \code{"ghi"});

\item each row in a matrix -- its own corresponding pattern:

<<>>=
(haystack <- matrix(  # example input
  do.call(stri_join,
    expand.grid(
      c("a", "b", "c"), c("a", "b", "c"), c("a", "b", "c")
    )), nrow=3))
needle <- c("a", "b", "c")
matrix(stri_count_fixed(haystack, needle),  # call to stringi
  nrow=3, dimnames=list(needle, NULL))
@

(this looks for \code{"a"} in the 1st row of
\code{haystack}, \code{"b"} in the 2nd row, and \code{"c"} in the 3rd;
in particular, there are 3 \code{"a"}s in \code{"aaa"}, 2 in \code{"aba"},
and 1 \code{"b"} in \code{"baa"};
this is possible due to the fact that matrices are
represented as ``flat'' vectors of length \code{nrow*ncol},
whose elements are read in a column-major (\proglang{Fortran}) order;
therefore, here,
pattern \code{"a"} is being sought in the 1st, 4th, 7th,
\dots{} string in \code{haystack}, i.e., \code{"aaa"}, \code{"aba"}, \code{"aca"}, \dots;
pattern \code{"b"} in the 2nd, 5th, 8th, \dots{} string;
and \code{"c"} in the 3rd, 6th, 9th, \dots{} one);

\medskip
On a side note, to match different patterns
with respect to each column, we can (amongst others)
apply matrix transpose twice (\code{t(stri_count_fixed(t(haystack), needle))}).



\item all strings -- all patterns:


<<>>=
haystack <- c("aaa", "bbb", "ccc", "abc", "cba", "aab", "bab", "acc")
needle <- c("a", "b", "c")
structure(
  outer(haystack, needle, stri_count_fixed),
  dimnames=list(haystack, needle))  # add row and column names
@

(which computes the counts over the Cartesian product
of the two arguments);



\end{itemize}






\subsection{Missing values}

Some base \proglang{R} string processing functions,
e.g., \code{paste()}, treat missing values as literal \code{"NA"}
strings.
\pkg{stringi}, however, does enforce the consistent
propagation of missing values (like arithmetic operations):

<<>>=
paste(c(NA_character_, "b", "c"), "x", 1:2)  # base R
stri_join(c(NA_character_, "b", "c"), "x", 1:2)  # stringi
@

For dealing with missing values, we may rely on the
convenience functions such as \code{stri_omit_na()} or
\code{stri_replace_na()}.



\subsection{Data flow}

All vector-like arguments (including factors and objects)
in \pkg{stringi} are treated in the same manner:
for example, if a function expects a character vector on input
and an object of other type is provided,
\code{as.character()} is called first
(we see that in the example above,
``\code{1:2}'' is treated as \code{c("1", "2")}).












Following \citep{Wickham2010:stringr}, \pkg{stringi}
makes sure the output data types are consistent
and that different functions are interoperable.
This makes operation chaining easier and less error prone.

For example, \code{stri_extract_first_regex()}
finds the first occurrence of a pattern in each string,
therefore the output is a character of the same length
as the input (with recycling rule in place if necessary).








<<>>=
haystack <- c("bacon", "spam", "jam, spam, bacon, and spam")
stri_extract_first_regex(haystack, "\\b\\w{1,4}\\b")
@

Note that a no-match (here, we have been looking for words
of at most 4 characters)
is marked with a missing string.
This makes the output vector size consistent with the length of
the inputs.

On the other hand, \code{stri_extract_all_regex()}
identifies all occurrences of a pattern, whose counts may differ from input to input,
therefore it yields a list of character vectors.

<<>>=
stri_extract_all_regex(haystack, "\\b\\w{1,4}\\b", omit_no_match=TRUE)
@

If the 3rd argument was not specified, a no-match
would be represented by a missing value (for consistency with the
previous function).


Also, care is taken so that the ``data'' or ``\code{x}''
argument is most often listed as the first one
(e.g., in base \proglang{R} we have \code{grepl(needle, haystack)}
vs \code{stri\_detect(haystack, needle)} here).
This makes the functions more intuitive to use,
but also more forward pipe operator-friendly
(either when using ``\code{|>}'' introduced in \proglang{R} 4.1
or ``\code{\%>\%}'' from \pkg{magrittr}).


Furthermore, for increased convenience, some functions have been added
despite the fact that they can be trivially reduced to a series of
other calls.
In particular, writing:

<<results="hide">>=
stri_sub_all(haystack,
  stri_locate_all_regex(haystack, "\\b\\w{1,4}\\b", omit_no_match=TRUE))
@

\noindent
yields the same result as in the previous example,
but refers to \code{haystack} twice.


\subsection[Further deviations from base R]{Further deviations from base \proglang{R}}

\pkg{stringi} can be used as a replacement of the existing string
processing functions. Also, it offers many
facilities not available in base \proglang{R}.
Except for being fully vectorised with respect to all crucial
arguments, propagating missing values and empty vectors consistently,
and following coherent naming conventions, our functions
deviate from their classic counterparts even further.






\paragraph{Following Unicode standards.}
Thanks to the comprehensive coverage of the most important
services provided by \pkg{ICU}, its users gain access to
collation, pattern searching, normalisation, transliteration, etc.,
that follow the recent Unicode standards for text processing
in any locale.
Due to this, as we state in Section~\ref{Sec:encoding},
all inputs are converted to Unicode and outputs are always
in UTF-8.



\paragraph{Portability issues in base \proglang{R}.}
As we have mentioned in the introduction,
base \proglang{R} string operations have traditionally been
limited in scope. There also might be some issues with regards to their portability,
reasons for which may be plentiful.
For instance, varied versions of the
\pkg{PCRE} (8.x or 10.x)
pattern matching libraries may be linked to
during the compilation of \proglang{R}. On Windows, there is a custom implementation
of \pkg{iconv} that has a set of character encoding
IDs not fully compatible with that on GNU/Linux: to select
the Polish locale, we are required to pass \code{"Polish\_Poland"}
to \code{Sys.setlocale()} on Windows whereas \code{"pl\_PL"} on
Linux.
Interestingly, \proglang{R} can be built against the system \pkg{ICU}
so that it uses its Collator for comparing strings (e.g., using the ``\code{<=}''
operator), however this is only optional and does not provide access to any
other Unicode services.























For example, let us consider the matching
of ``all letters'' by means of the built-in \code{gregexpr()} function
and the \pkg{TRE} (\code{perl=FALSE})
and \pkg{PCRE} (\code{perl=TRUE}) libraries
using a POSIX-like and Unicode-style character set
(see Section~\ref{Sec:regex} for more details):

\begin{Schunk}
\begin{Sinput}
R> x <- "AEZaezĄĘŻąęż"  # "AEZaez\u0104\u0118\u017b\u0105\u0119\u017c"
R> stri_sub(x, gregexpr("[[:alpha:]]", x, perl=FALSE)[[1]], length=1)
R> stri_sub(x, gregexpr("[[:alpha:]]", x, perl=TRUE)[[1]],  length=1)
R> stri_sub(x, gregexpr("\\p{L}", x, perl=TRUE)[[1]],       length=1)
\end{Sinput}
\end{Schunk}

On Ubuntu Linux 20.04 (UTF-8 locale), the respective outputs are:

\begin{Schunk}
\begin{Soutput}
[1] "A" "E" "Z" "a" "e" "z" "Ą" "Ę" "Ż" "ą" "ę" "ż"
[1] "A" "E" "Z" "a" "e" "z"
[1] "A" "E" "Z" "a" "e" "z" "Ą" "Ę" "Ż" "ą" "ę" "ż"
\end{Soutput}
\end{Schunk}

On Windows, when \code{x} is marked as UTF-8
(see Section \ref{Sec:encoding}), the author obtained:

\begin{Schunk}
\begin{Soutput}
[1] "A" "E" "Z" "a" "e" "z"
[1] "A" "E" "Z" "a" "e" "z"
[1] "A" "E" "Z" "a" "e" "z" "Ą" "Ę" "Ż" "ą" "ę" "ż"
\end{Soutput}
\end{Schunk}

And again on Windows using the Polish locale
but \code{x} marked as natively-encoded (CP-1250 in this case):

\begin{Schunk}
\begin{Soutput}
[1] "A" "E" "Z" "a" "e" "z" "Ę" "ę"
[1] "A" "E" "Z" "a" "e" "z" "Ą" "Ę" "Ż" "ą" "ę" "ż"
[1] "A" "E" "Z" "a" "e" "z" "Ę" "ę"
\end{Soutput}
\end{Schunk}

As we mention in Section~\ref{Sec:collator},
when \pkg{stringi} links to \pkg{ICU} built from sources
(\code{install.pa\-ckages("stringi", configure.args="--disable-pkg-config")}),
we are always guaranteed to get the same results on every platform.




\paragraph{High performance of \pkg{stringi}.}
Because of the aforementioned reasons, functions in \pkg{stringi}
do not refer to their  base \proglang{R} counterparts.
The operations that do not rely on \pkg{ICU} services
have been rewritten from scratch with speed and portability in mind.
For example, here are some timings of string concatenation:


<<performance1,cache=TRUE>>=
x <- stri_rand_strings(length(LETTERS) * 1000, 1000)
microbenchmark::microbenchmark(
  join2=stri_join(LETTERS, x, sep="", collapse=", "),
  join3=stri_join(x, LETTERS, x, sep="", collapse=", "),
  r_paste2=paste(LETTERS, x, sep="", collapse=", "),
  r_paste3=paste(x, LETTERS, x, sep="", collapse=", ")
)
@

Another example -- timings of fixed pattern searching:

<<performance2,echo=-1,cache=TRUE>>=
set.seed(123)
x <- stri_rand_strings(100, 100000, "[actg]")
y <- "acca"
microbenchmark::microbenchmark(
  fixed=stri_locate_all_fixed(x, y),
  regex=stri_locate_all_regex(x, y),
  coll=stri_locate_all_coll(x, y),
  r_tre=gregexpr(y, x),
  r_pcre=gregexpr(y, x, perl=TRUE),
  r_fixed=gregexpr(y, x, fixed=TRUE)
)
@


\paragraph{Different default arguments and greater configurability.}
Some functions in \pkg{stringi} have different,
more natural default arguments,
e.g., \code{paste()} has \code{sep=" "} but
\code{stri_join()} has \code{sep=""}.
Also, as there is no one-fits-all solution to all problems,
many arguments have been introduced for more detailed tuning.

\paragraph{Preserving attributes.}
Generally, \pkg{stringi} preserves no object attributes
whatsoever, but a user can make sure themself
that this is becomes the case, e.g., by calling
``\code{x[] <- stri_...(x, ...)}''
or ``\code{`attributes<-`(stri_...(x, ...), attributes(x))}''.










\section{Basic string operations}\label{Sec:basic}

Let us proceed with a detailed description of the most important
facilities in the \pkg{stringi} package that might be of interest to
the broad statistical and data analysis audience.



\subsection{Computing length and width}


First we shall review the functions related to determining
the number of entities in each string.



Let us consider the following character vector:

<<>>=
x <- c("spam", "你好", "\u200b\u200b\u200b", NA_character_, "")
@

The \code{x} object consists of 5 character strings:

<<>>=
length(x)
@


\code{stri_length()} computes the length of each string.
More precisely, the function gives the number of Unicode code points
in each string, see Section~\ref{Sec:codepoints} for more details.

<<>>=
stri_length(x)
@

\noindent
The first string carries 4 ASCII (English) letters,
the second consists of 2 Chinese characters (U+4F60, U+597D; a greeting),
and the third one is comprised of 3 zero-width spaces (U+200B).
Note that the 5th element in \code{x} is an empty string, \code{""},
hence its length is  0.
Moreover, there is a missing (\code{NA}) value
at index 4, therefore the corresponding length is undefined as well.




When formatting strings for display
(e.g., in a report dynamically generated with \code{Sweave()}
or \pkg{knitr}; see \citealp{knitr}),
a string's width estimate may be more informative --
an approximate number of text columns it will occupy when printed
using a monospaced font.
In particular, many Chinese, Japanese, Korean, and most emoji
characters take up two text cells. Some code points, on the other hand,
might be of width 0 (e.g., the above ZERO WIDTH SPACE, U+200B).



<<>>=
stri_width(x)
@

\noindent














\subsection{Joining}

Below we describe the functions that are related to string concatenation.


\paragraph{Operator \code{\%s+\%}.}
To join  the corresponding strings in two character vectors,
we may use the binary \code{\%s+\%} operator:

<<>>=
x <- c("tasty", "delicious", "yummy", NA)
x %s+% " " %s+% c("spam", "bacon")
@






\paragraph{Flattening.}
The elements in a character vector can be joined (``aggregated'')
to form a single string via a call to  \code{stri\_flatten()}:

<<>>=
stri_flatten(stri_omit_na(x), collapse=", ")
@

\noindent
Note that the token separator, given by the \code{collapse} argument,
defaults to the empty string.



\paragraph{Generalisation.}
Both the \code{\%s+\%} operator and the \code{stri\_flatten()} function
are generalised by
\code{stri\_join()} (alias: \code{stri\_paste()}, \code{stri\_c()}):

<<>>=
stri_join(c("X", "Y", "Z"), 1:6, "a")  # sep="", collapse=NULL
@

By default, the \code{sep} argument, which controls how corresponding
strings are delimited, is set to the empty string
(like in the base \code{paste0()} but unlike in \code{paste()}).
Moreover, \code{collapse} is \code{NULL}, which means that
the resulting outputs will not be joined to form a single string.
This can be changed if need be:

<<>>=
stri_join(c("X", "Y", "Z"), 1:6, "a", sep="_", collapse=", ")
@



\noindent
Note how the two (1st, 3rd) shorter vectors were {recycled} to match
the longest (2nd) vector's length. The latter was of numeric type,
but it was implicitly coerced via a call to \code{as.character()}.








\paragraph{Duplicating.}
To duplicate given strings, we call
\code{stri\_dup()} or the \code{\%s*\%} operator:


<<>>=
stri_dup(letters[1:5], 1:5)
@

The above is synonymous with \code{letters[1:5] \%s*\% 1:5}.


\paragraph{Within-list joining.}
There is also a convenience function that applies \code{stri\_flatten()}
on each character vector in a given list:

<<>>=
words <- list(c("spam", "bacon", "sausage", "spam"), c("eggs", "spam"))
stri_join_list(words, sep=", ")  # collapse=NULL
@



\noindent
This way, a list of character vectors can be converted to
a character vector. Such sequences of variable length sequences of
strings are generated by, amongst others,
\code{stri_sub_all()} and \code{stri_extract_all()}.






\subsection{Extracting and replacing substrings}

Next group of functions deals with the extraction and replacement
of particular sequences of code points in given strings.

\paragraph{Indexing vectors.}
Recall that in order to select a subsequence from any \proglang{R} vector,
we use the square-bracket operator\footnote{More precisely, \code{x[i]}
is a syntactic sugar for a call to \code{`[`(x, i)}.
Moreover, if \code{x} is a list, \code{x[[i]]} can be used to
extract its \code{i}-th element (alias \code{`[[`(x, i)}).
Knowing the ``functional'' form of the operators allows us to, for instance,
extract all first elements from each vector in a list
by simply calling \code{sapply(x, "[[", 1)}.}
with an index vector consisting of either
non-negative integers, negative integers,
or logical values\footnote{If an object's \code{names} attribute is set,
indexing with a character vector is also possible.}.

For example, here is how to select specific elements in a vector:

<<>>=
x <- c("spam", "buckwheat", "", NA, "bacon")
x[1:3]                           # from 1st to 3rd string
x[c(1, length(x))]               # 1st and last
@

\noindent
Exclusion of elements at specific positions can be performed like:

<<>>=
x[-1]                            # all but 1st
@

\noindent
Filtering based on a logical vector can be used to extract
strings fulfilling desired criteria:

<<>>=
x[!stri_isempty(x) & !is.na(x)]
@


\paragraph{Extracting substrings.}
A character vector is, in its very own essence, a sequence of
sequences of code points.
To extract specific substrings from each string in a collection,
we can use the \code{stri\_sub()} function.

<<>>=
y <- "spam, egg, spam, spam, bacon, and spam"
stri_sub(y, 18)             # from 18th code point to end
stri_sub(y, 12, to=15)      # from 12th to 15th code point (inclusive)
@

Negative indices count from the end of a string.

<<>>=
stri_sub(y, -15, length=5)  # 5 code points from 15th last
@




\paragraph{\code{stri\_sub\_all()} function.}
If some deeper vectorisation level is necessary, \code{stri_sub_all()}
comes in handy. It extracts multiple (possibly different) substrings
from all the strings provided:


<<>>=
(z <- stri_sub_all(
              c("spam",     "bacon", "sorghum"),
  from   = list(c(1, 3, 4), -3,      c(2, 4)),
  length = list(1,           3,      c(4, 3))))
@

As the number of substrings to extract from each string might vary,
the result is a list of character strings.
We have obtained:
substrings of length 1 starting at positions 1, 3, and 4 in \code{x[1]},
then a length-3 substring that starts at the 3rd code point
from the end of \code{x[2]},
and length-4 and -3 substrings starting at, respectively,
the 2nd and 4th code point
of \code{x[3]} (where \code{x} denotes the subsetted vector).











\paragraph{``From--to'' and ``from--length'' matrices.}
The second parameter of both \code{stri_sub()} and \code{stri_sub_list()}
can also be fed with a two-column matrix
of the form \code{cbind(from, to)}. Here, the first column
gives the start indices and the second column defines the end ones.
Such matrices are generated, amongst others, by the \code{stri_locate_*()}
functions (see below for details).



<<>>=
(from_to <- matrix(1:8, ncol=2, byrow=TRUE))
stri_sub(c("abcdefgh", "ijklmnop"), from_to)
@

\noindent
Due to recycling, this has extracted
elements at positions 1:2 from the 1st string,
at 3:4 from the 2nd one, 5:6 from the 1st, and 7:8 from the 2nd again.

Note the difference between the above output and the following one:

<<>>=
stri_sub_all(c("abcdefgh", "ijklmnop"), from_to)
@

This time, we extract four identical sections from each of the two inputs.


\medskip
Moreover, if the second column of the index matrix is named
\code{"length"} (and only if this is exactly the case),
i.e., the indexer is of the form \code{cbind(from, length=length)},
extraction will be based on the extracted chunk size.






\paragraph{Replacing substrings.}
\code{stri\_sub\_replace()} returns a version
of a character vector with some chunks replaced by other
strings:

<<>>=
stri_sub_replace(c("abcde", "ABCDE"),
  from=c(2, 4), length=c(1, 2), replacement=c("X", "uvw"))
@

\noindent
The above replaced ``\code{b}'' (the length-1 substring
starting at index 2 of the 1st string) with ``\code{X}''
and ``\code{DE}''
(the length-2 substring at index 4 of the 2nd string)
with ``\code{uvw}''.


Similarly, \code{stri\_sub\_replace\_all()} replaces
multiple substrings within each string in a character vector:

<<>>=
stri_sub_replace_all(
                   c("abcde",  "ABCDE"),
  from        = list(c(2, 4),  c(0,    3,   6)),
  length      = list(  1,      c(0,    2,   0)),
  replacement = list(  "Z",    c("uu", "v", "wwww")))
@

\noindent
Note how we have obtained the insertion of new content
at the start and the end of the 2nd input.


\paragraph{Replacing substrings in-place.}
The corresponding {replacement functions} modify
a character vector in-place:

<<>>=
y <- "spam, egg, spam, spam, bacon, and spam"
stri_sub(y, 7, length=3) <- "spam"  # in-place replacement, egg → spam
print(y)                            # y has changed
@

Note that the state of \code{y} has changed in such a way that the
substring of length 3 starting at the 7th code point
was replaced by a length-4 content.

Many replacements within a single string are also possible:

<<>>=
y <- "aa bb cc"
stri_sub_all(y, c(1, 4, 7), length=2) <- c("A", "BB", "CCC")
print(y)                            # y has changed
@

\noindent
This has replaced 3 length-2 chunks within \code{y} with new content.





\section{Code-pointwise comparing}\label{Sec:fixed}

There are many circumstances where we are faced with testing
whether two strings (or parts thereof)
consist  of exactly the same Unicode code points, in exactly the same order.
These include, for instance, matching a nucleotide sequence
in a DNA profile and querying for system resources based on file names or UUIDs.
Such tasks, due to their simplicity, can be performed very efficiently.


\subsection{Testing for equality of strings}



To quickly test whether the corresponding strings in two character vectors
are identical (in a code-pointwise manner), we can use the \code{\%s===\%}
operator or, equivalently, the \code{stri\_cmp\_eq()} function.
Moreover, \code{\%s!==\%} and \code{stri\_cmp\_neq()}
implement the not-equal-to relation.

<<>>=
"actg" %s===% c("ACTG", "actg", "act", "actga", NA)
@

Due to recycling, the first string was compared against the
5 strings in the 2nd operand. There is only 1 exact match.



\begin{table}[b!]
\centering
\begin{tabularx}{1.0\linewidth}{p{4.8cm}X}
\toprule
\bfseries Name(s) & \bfseries Meaning \\
\midrule
\code{stri\_count()} &  count pattern matches    \\
\midrule
\code{stri\_detect()} & detect pattern matches     \\
\midrule
\code{stri\_endswith()} &  [all but \code{regex}] detect pattern matches at end of string  \\
\midrule
\code{stri\_extract\_all()}, \code{stri\_extract\_first()}, \code{stri\_extract\_last()}  & extract pattern matches     \\
\midrule
\code{stri\_locate\_all()}, \code{stri\_locate\_first()}, \code{stri\_locate\_last()}  & locate pattern matches     \\
\midrule
\code{stri\_match\_all()}, \code{stri\_match\_first()}, \code{stri\_match\_last()}   &  [\code{regex} only] extract matches to regex capture groups   \\
\midrule
\code{stri\_replace\_all()}, \code{stri\_replace\_first()}, \code{stri\_replace\_last()}  &     substitute pattern matches with some replacement strings \\
\midrule
\code{stri\_split()}  & split up a string at pattern matches     \\
\midrule
\code{stri\_startswith()}  &  [all but \code{regex}] detect pattern matches at start of string   \\
\midrule
\code{stri\_subset()}, \code{`stri\_subset<-`()}  & return or replace strings
that contain pattern matches \\
\bottomrule
\end{tabularx}

\caption{\label{Tab:searchfuns} String search/pattern matching functions in \pkg{stringi}.
Each function, unless otherwise indicated, can be used in conjunction
with any search engine, e.g., we have
\code{stri\_count\_fixed()} (see Section~\ref{Sec:fixed}),
\code{stri\_detect\_regex()} (see Section~\ref{Sec:regex}), and
\code{stri\_split\_coll()} (see Section~\ref{Sec:collator}).}
\end{table}


\subsection{Searching for fixed strings}


For detecting if a string contains
a given fixed substring  (code-pointwisely),
the fast KMP \citep{KnuthETAL1977:kmp} algorithm, with worst time complexity of $O(n+p)$
(where $n$ is the length of the string and $p$ is the length of the pattern),
has been implemented in \pkg{stringi} (with numerous tweaks
for even faster matching).

Table~\ref{Tab:searchfuns} lists the string search functions available
in \pkg{stringi}.  Below we explain their behaviour in the context of fixed
 pattern matching. Notably, their description
is quite detailed, because -- as we shall soon find out --
the corresponding operations are available for the two other search
engines: based on regular expressions and the \pkg{ICU} Collator,
see Section~\ref{Sec:regex} and Section~\ref{Sec:collator}.






\subsection{Counting matches}

The \code{stri_count_fixed()} function counts the number of
times a fixed pattern occurs in a given string.

<<>>=
stri_count_fixed("abcabcdefabcabcabdc", "abc")  # search pattern is "abc"
@







\subsection{Search engine options}

The pattern matching engine may be tuned up by passing
further arguments to the search functions (via ``\code{...}'';
they are redirected as-is to \code{stri_opts_fixed()}).
Table~\ref{Tab:fixed_opts} gives the list of available options.





First, we may switch on the simplistic\footnote{Which is not suitable
for real-world NLP tasks, as it assumes
that changing the case of a single code point always produces one and only
one item;
This way, \code{"groß"} does not compare equal to \code{"GROSS"},
see Section~\ref{Sec:collator} (and partially Section~\ref{Sec:regex}) for a workaround.}
case-insensitive matching.


<<>>=
stri_count_fixed("ACTGACGacgggACg", "acg", case_insensitive=TRUE)
@

Second, we can indicate our interest in detecting
overlapping pattern matches or whether searching should continue
at the end of each match
(the latter being the default behaviour):

<<>>=
stri_count_fixed("acatgacaca", "aca")  # overlap=FALSE (default)
stri_count_fixed("acatgacaca", "aca", overlap=TRUE)
@



\begin{table}[t!]
\centering

\begin{tabularx}{1.0\linewidth}{p{4cm}X}
\toprule
\bfseries{Option}            &\bfseries Purpose \\
\midrule
\code{case\_insensitive} & logical; whether to enable the simple
case-insensitive matching (defaults to \code{FALSE}) \\
\midrule
\code{overlap} & logical; whether to enable the detection of overlapping
matches (defaults to \code{FALSE});
available in \code{stri\_extract\_all\_fixed()},
\code{stri\_locate\_all\_fixed()}, and \code{stri\_count\_fixed()}
\\
\bottomrule
\end{tabularx}

\caption{\label{Tab:fixed_opts} Options for the fixed pattern search
engine, see \code{stri\_opts\_fixed()}.}
\end{table}



\subsection{Detecting and subsetting patterns}

A somewhat simplified version of the above search task involves asking
whether a pattern occurs in a string at all. Such an operation can be
performed with a call to \code{stri\_detect\_fixed()}.

<<>>=
x <- c("abc", "abcd", "def", "xyzabc", "uabdc", "dab", NA, "abc")
stri_detect_fixed(x, "abc")
@

We can also indicate that a no-match is rather of our  interest
by passing \code{negate=TRUE}.
What is more, there is an option to stop searching
once a given number of matches has been found
in the \code{haystack} vector (as a whole),
which can speed up the processing of larger data sets:

<<>>=
stri_detect_fixed(x, "abc", negate=TRUE, max_count=2)
@

\noindent
This can be useful in scenarios such as ``find the first 2 matching
resource IDs''.


\medskip
There are also functions that verify whether a string
starts or ends\footnote{Note that testing for a pattern match at the start
or end of a string has not been implemented separately for \code{regex} patterns,
which support \code{"\textasciicircum"} and \code{"\$"} anchors that  serve exactly this very purpose.}
with a pattern match:

<<>>=
stri_startswith_fixed(x, "abc")  # from=1 - match at start
stri_endswith_fixed(x, "abc")    # to=-1 - match at end
@


\medskip
Pattern detection is often performed in conjunction
with character vector subsetting.
This is why we have a specialised (and hence slightly faster)
function that  returns only the strings that match a given pattern.


<<>>=
stri_subset_fixed(x, "abc", omit_na=TRUE)
@

The above is equivalent to \code{x[which(stri_detect_fixed(x, "abc"))]}
(note the argument responsible for the removal of missing values),
but avoids writing \code{x} twice.
It hence is particularly convenient when \code{x} is generated
programmatically on the fly, using some complicated expression.
Also, it works well with the forward pipe operator, as we can write
``\code{x |> stri_subset_fixed("abc", omit_na=TRUE)}''.




There is also a replacement version of this function:

<<>>=
stri_subset_fixed(x, "abc") <- c("*****", "***")  # modifies x in-place
print(x)  # x has changed
@






\subsection{Locating and extracting patterns}

The functions from the \code{stri_locate()} family
aim to pinpoint the positions of pattern matches.
First, we may be interested in getting to know the location of the
first or the last pattern occurrence:

<<>>=
x <- c("aga", "actg", NA, "AGagaGAgaga")
stri_locate_first_fixed(x, "aga")
stri_locate_last_fixed(x, "aga", get_length=TRUE)
@

\noindent
In both examples we obtain a two-column matrix
with the number of rows determined by the recycling rule (here:
the length of \code{x}).
In the former case, we get a ``from--to'' matrix (\code{get_length=FALSE};
the default) where missing values correspond to either missing inputs or
no-matches. The latter gives a ``from--length''-type matrix,
where negative lengths correspond to the not-founds.

Second, we may be yearning for the locations of all the matching
substrings. As the number of possible answers may vary from string to string,
the result is a list of index matrices.

<<>>=
stri_locate_all_fixed(x, "aga", overlap=TRUE, case_insensitive=TRUE)
@

\noindent
Note again that a no-match is indicated by a single-row matrix
with two missing values (or with negative length if \code{get_length=TRUE}).
This behaviour can be changed by setting the \code{omit_no_match}
argument to \code{TRUE}.


\medskip
Let us recall that ``from--to'' and ``from--length''
matrices of the above kind constitute particularly
fine inputs to \code{stri\_sub()} and \code{stri\_sub\_all()}.
However, if merely the extraction of the matching substrings is needed,
it will be more convenient to rely on the functions from the
\code{stri\_extract()} family:

<<>>=
stri_extract_first_fixed(x, "aga", case_insensitive=TRUE)
stri_extract_all_fixed(x, "aga",
  overlap=TRUE, case_insensitive=TRUE, omit_no_match=TRUE)
@






\subsection{Replacing pattern occurrences}

In order to replace each match with a corresponding
replacement string, we can refer to \code{stri\_replace\_all()}:

<<>>=
x <- c("aga", "actg", NA, "ggAGAGAgaGAca", "agagagaga")
stri_replace_all_fixed(x, "aga", "~", case_insensitive=TRUE)
@

Note that the inputs that are not part of any match are left unchanged.
The input object is left unchanged, because it is not a replacement
function per se.

The operation is vectorised with respect to all the three arguments
(haystack, needle, replacement string),
with the usual recycling behaviour if necessary.
If a different arguments' vectorisation scheme is required,
we can set the \code{vectorise_all} argument of \code{stri_replace_all()}
to \code{FALSE}.
Compare the following:

<<>>=
stri_replace_all_fixed("The quick brown fox jumped over the lazy dog.",
  c("quick", "brown",      "fox", "lazy",    "dog"),
  c("slow",  "yellow-ish", "hen", "spamity", "llama"))
stri_replace_all_fixed("The quick brown fox jumped over the lazy dog.",
  c("quick", "brown",      "fox", "lazy", "dog"),
  c("slow",  "yellow-ish", "hen", "spamity", "llama"),
  vectorise_all=FALSE)
@

\noindent
Here, for every string in the \code{haystack}, we observe the vectorisation
independently over the \code{needles} and replacement strings.
Each occurrence of the 1st needle is superseded by the 1st replacement
string, then the search is repeated for the 2nd needle so as to replace
it with the 2nd corresponding replacement string, and so forth.

Moreover, \code{stri_replace_first()} and \code{stri_replace_last()}
can identify and replace the first and the last match, respectively.




\subsection{Splitting}

To  split each element in the \code{haystack} into substrings,
where the \code{needles} define the delimiters that separate
the inputs into tokens,
we call \code{stri\_split()}:

<<>>=
x <- c("a,b,c,d", "e", "", NA, "f,g,,,h,i,,j,")
stri_split_fixed(x, ",", omit_empty=TRUE)
@

The result is a list of character vectors, as each string
in the \code{haystack}
might be split into a possibly different number of tokens.

There is also an option to limit the number of tokens
(parameter \code{n}).










\section{Regular expressions}\label{Sec:regex}

Regular expressions (regexes) provide us with a concise
grammar for defining systematic patterns which can be sought in character
strings. Examples of such patterns include:
specific fixed substrings,
emojis of any kind,
stand-alone sequences of lower-case Latin letters (``words''),
substrings that can be interpreted as real numbers (with or without fractional part, also in scientific notation),
telephone numbers,
email addresses, or
URLs.

Theoretically, the concept of regular pattern matching
dates back to the so-called regular languages and finite state
automata \citep{kleene},
see also \citep{hopcroftullman,automata}.
Regexes in the form as we know today have already been present
in one of the pre-Unix implementations of the command-line text
editor \pkg{qed} (\citealp{qed}; the predecessor of the well-known \pkg{sed}).

Base  \proglang{R} gives access to two different regex matching engines
(via functions such as \code{gregexpr()} and \code{grep()},
see Table~\ref{Tab:oldstringr}):
\begin{itemize}
\item {ERE}\footnote{Via the \pkg{TRE} library
(\url{https://github.com/laurikari/tre/}).}
(extended regular expressions that conform
to the POSIX.2-1992 standard);
used by default,
\item {PCRE}\footnote{Via the \pkg{PCRE2}
library (\url{https://www.pcre.org/}).}
(\proglang{Perl}-compatible regular expressions);
activated when \code{perl=TRUE} is set.
\end{itemize}
Other matchers are implemented in the \pkg{ore}
(\citealp{ore}; via the \pkg{Onigmo} library)
and \pkg{re2r} (\citealp{re2r}; \pkg{RE2}) packages.


\pkg{Stringi}, on the other hand, provides access to the regex engine
implemented in \pkg{ICU}, which was inspired
by \proglang{Java}'s \pkg{util.regex}
in \pkg{JDK 1.4}. Their syntax is mostly compatible with that of \pkg{PCRE},
although certain more advanced facets might not be supported (e.g., recursive
patterns). On the other hand, \pkg{ICU} regexes fully conform to the
Unicode Technical Standard \#18 \citep{uts18:regex} and hence provide
comprehensive support for Unicode.


It is worth noting that most programming languages
as well as advanced text editors and development environments (including
\pkg{Kate}, \pkg{Eclipse}, \pkg{VSCode}, and \pkg{RStudio})
support finding or replacing patterns with regexes.
Therefore, they should be amongst the instruments
at every data scientist's disposal.
One general introduction to regexes is \citep{friedl}.
The \pkg{ICU} flavour is summarised at
\url{https://unicode-org.github.io/icu/userguide/strings/regexp.html}.






Below we provide a concise yet comprehensive introduction
to the topic from the perspective of the \pkg{stringi} package users.
This time we will use the pattern search routines whose names
end with the \code{*_regex()} suffix.
Apart from \code{stri_detect_regex()}, \code{stri_locate_all_regex()},
and so forth, in Section~\ref{Sec:Capturing} we introduce
\code{stri_match_all_regex()}.
Moreover, Table~\ref{Tab:regex_opts} lists the available options
for the regex engine.






\begin{table}[t!]
\centering

\begin{tabularx}{1.0\linewidth}{p{4.6cm}X}
\toprule
\bfseries{Option}            &\bfseries Purpose \\
\midrule
\code{case\_insensitive}\newline
[regex flag \code{(?i)}] & logical; defaults to \code{FALSE}; whether to enable (full) case-insensitive matching  \\
\midrule
\code{comments}\newline
[regex flag \code{(?x)}] & logical; defaults to \code{FALSE}; whether to allow white spaces and comments within patterns  \\
\midrule
\code{dot\_all}\newline
[regex flag \code{(?s)}] & logical; defaults to \code{FALSE}; if set, ``\code{.}'' matches line terminators; otherwise its matching stops at a line end  \\
\midrule
\code{literal} & logical; defaults to \code{FALSE}; whether to treat the entire pattern as a literal string; note that in most cases the code-pointwise string search facilities
(\code{*\_fixed()} functions described in Section~\ref{Sec:fixed}) are faster
\\
\midrule
\code{multi\_line}\newline
[regex flag \code{(?m)}] & logical; defaults to \code{FALSE}; if set, ``\code{\$}'' and ``\code{\textasciicircum}'' recognise line terminators within a string; otherwise, they match only at start and end of the input \\
\midrule
\code{unix\_lines} & logical; defaults to \code{FALSE}; when enabled, only the Unix line ending, i.e., U+000A, is honoured as a terminator by ``\code{.}'', ``\code{\$}'', and ``\code{\textasciicircum}''\\
\midrule
\code{uword}\newline
[regex flag \code{(?w)}] & logical; defaults to \code{FALSE}; whether to use the Unicode definition of word boundaries (see Section~\ref{Sec:BoundaryAnalysis}), which are quite different from the traditional regex word boundaries\\
\midrule
\code{error\_on\_unknown\_escapes} & logical; defaults to \code{FALSE}; whether unrecognised backslash-escaped characters trigger an error; by default,  unknown backslash-escaped ASCII letters represent themselves \\
\midrule
\code{time\_limit} & integer; processing time limit for match operations in $\sim$milliseconds
(depends on the CPU speed);
0 for no limit (the default) \\
\midrule
\code{stack\_limit} & integer; maximal size, in bytes, of the heap storage available
for the matcher's backtracking stack; setting a limit is desirable if poorly
written regexes are expected on input; 0 for no limit (the default) \\
\bottomrule
\end{tabularx}

\caption{\label{Tab:regex_opts} Options for the regular expressions
search engine, see \code{stri\_opts\_regex()}.}
\end{table}



\subsection{Matching individual characters}\label{Sec:RegexIndividualChars}

We begin by discussing different ways to define character sets.
In this part, determining the length of all matching substrings
will be quite straightforward.

The following characters have special
meaning to the regex engine:
\[
\text{
  \texttt{.}\quad
  \textbackslash{}\quad
  \texttt{|}\quad
  \texttt{(}\quad
  \texttt{)}\quad
  \texttt{[}\quad
  \texttt{\{}\quad
  \texttt{\}}\quad
  \texttt{\^{}}\quad
  \texttt{\$}\quad
  \texttt{*}\quad
  \texttt{+}\quad
  \texttt{?}\quad
}
\]

Any regular expression that does not contain the above
behaves like a fixed pattern:

<<>>=
stri_count_regex("spam, eggs, spam, bacon, sausage, and spam", "spam")
@

There are hence 3 occurrences of a pattern that is comprised
of 4 code points, ``\code{s}'' followed by ``\code{p}'',
then by ``\code{a}'', and ending with ``\code{m}''.

However, this time the case insensitive mode fully
supports Unicode matching\footnote{%
This does not mean, though, that it considers canonically
equivalent strings as equal,
see Section~\ref{Sec:Equivalence} for a discussion and a workaround.}:

<<>>=
stri_detect_regex("groß", "GROSS", case_insensitive=TRUE)
@

If we wish to include a special character as part of a regular expression --
so that it is treated literally -- we will need to escape it with  a backslash,
``\textbackslash''. Yet, the backlash itself
has a special meaning to \proglang{R}, see \code{help("Quotes")},
therefore it needs to be preceded by another backslash.

<<>>=
stri_count_regex("spam...", "\\.")   # "\\" is a way to input a single \
@

In other words, the \proglang{R} string \code{"\textbackslash\textbackslash."}
is seen by the regex engine as ``\code{\textbackslash.}'' and interpreted
as the dot character (literally). Alternatively,
since \proglang{R} 4.0 we can also input the so-called literal strings
like \code{r"(\textbackslash.)"}.


\paragraph{Matching any character.}
The (unescaped) dot, ``\code{.}'', matches any code point except the newline.

<<>>=
x <- "Ham, spam,\njam, SPAM, eggs, and spam"
stri_extract_all_regex(x, "..am", case_insensitive=TRUE)
@

The above matches non-overlapping length-4 substrings that
end with ``\code{am}''.

The dot's insensitivity to the newline character is motivated
by the need to maintain the compatibility with tools such as \pkg{grep}
(when searching within text files in a line-by-line manner).
This behaviour can be altered by setting the \code{dot_all} option to \code{TRUE}.

<<>>=
stri_extract_all_regex(x, "..am", dot_all=TRUE, case_insensitive=TRUE)
@



\paragraph{Defining character sets.}
Sets of characters can be introduced by enumerating
their members within a pair of
square brackets.
For instance, ``\code{[abc]}'' denotes the set $\{\mathtt{a},\mathtt{b},\mathtt{c}\}$
-- such a regular expression matches one (and only one) symbol from this set.
Moreover, in:


<<>>=
stri_extract_all_regex(x, "[hj]am")
@


\noindent
the ``\code{[hj]am}'' regex matches:
``\code{h}'' or ``\code{j}'', followed by ``\code{a}'', followed by ``\code{m}''.
In other words, \code{"ham"} and \code{"jam"}
are the only two strings that are matched by this pattern
(unless matching is done case-insensitively).


The following characters, if used within square brackets, may be treated
non-literally:
\[
\text{
  \textbackslash{}\quad
  \texttt{[}\quad
  \texttt{]}\quad
  \texttt{\^{}}\quad
  \texttt{{}-{}}\quad
  \texttt{\&}\quad
}
\]
Therefore, to include them as-is in a character set, the
backslash-escape must be used.
For example, ``\code{[\textbackslash[\textbackslash]\textbackslash\textbackslash]}'' matches
the backslash or a square bracket.

\paragraph{Complementing sets.}
Including ``\code{\^{}}'' after the opening square bracket denotes the set complement.
Hence, ``\code{[\^{}abc]}'' matches any code point except ``\code{a}'',
``\code{b}'', and ``\code{c}''.
Here is an example where we seek any substring that consists of 3 non-spaces.

<<>>=
x <- "Nobody expects the Spanish Inquisition!"
stri_extract_all_regex(x, "[^ ][^ ][^ ]")
@



\paragraph{Defining Code Point Ranges.}
Each Unicode code point can be referenced by its unique numeric identifier,
see Section~\ref{Sec:codepoints}  for more details.
For instance, ``\code{a}'' is assigned code U+0061 and ``\code{z}'' is mapped to U+007A.
In the pre-Unicode era (mostly with regards to the ASCII codes, $\le$ U+007F,
representing English letters, decimal digits, some punctuation characters,
and a few control characters),
we were used to relying on specific code ranges; e.g.,
``\code{[a-z]}'' denotes the set comprised of all
characters with codes between U+0061 and U+007A, i.e., lowercase letters
of the English (Latin) alphabet.

<<>>=
stri_extract_all_regex("In 2020, Gągolewski had fun once.", "[0-9A-Za-z]")
@



The above pattern denotes a union of 3 code ranges:
digits and ASCII upper- and lowercase letters.

Nowadays, in the processing of text in natural languages, this
notation should rather be avoided. Note the missing ``\code{ą}''
(Polish ``\code{a}'' with ogonek) in the result.


\paragraph{Using predefined character sets.}
Each code point is assigned a unique general category,
which can be thought of as a character's class,
see \citep{usa44:ucd}.
Sets of characters from each category can be referred to,
amongst others, by using the ``\code{\textbackslash{}p\{category\}}''
(or, equivalently, ``\code{[\textbackslash{}p\{category\}]}'') syntax:

<<>>=
x <- "aąbßÆAĄB你123,.;'! \t-+=[]©←→”„²³¾"
p <- c("\\p{L}", "\\p{Ll}", "\\p{Lu}", "\\p{N}", "\\p{P}", "\\p{S}")
structure(stri_extract_all_regex(x, p), names=p)
@

The above yield a match to: arbitrary letters, lowercase letters, uppercase letters,
numbers, punctuation marks, and symbols, respectively.


Characters' binary properties and scripts can also be referenced in a similar manner.
Some other noteworthy classes include:

<<>>=
p <- c("\\w", "\\d", "\\s")
structure(stri_extract_all_regex(x, p), names=p)
@

These give: word characters, decimal digits (``\code{\textbackslash{}p\{Nd\}}''),
and spaces (``\code{[\textbackslash{}t\textbackslash{}n\textbackslash{}f\textbackslash{}r\textbackslash{}p\{Z\}]}''),
in this order.

Moreover, e.g., the upper-cased ``\code{\textbackslash{}P\{category\}}'' and
``\code{\textbackslash{}W}'' are equivalent to
``\code{[\^{}\textbackslash{}p\{category\}]}'' and
``\code{[\^{}\textbackslash{}w]}'', respectively, i.e.,
denote their complements.




\paragraph{Avoiding POSIX classes.}
The use of the POSIX-like character classes should be avoided,
because they are generally not well-defined.

In particular, in POSIX-like regex engines, ``\code{[:punct:]}''
stands for the character class corresponding to the \code{ispunct()}
function in \proglang{C} (see ``\code{man 3 ispunct}'' on Unix-like systems).
According to ISO/IEC 9899:1990 (ISO C90), \code{ispunct()} tests for
any printing character except for the space or a character for which \code{isalnum()}
is true.

Base \proglang{R} with \pkg{PCRE} yields on the current
author's machine:

<<>>=
x <- ",./|\\<>?;:'\"[]{}-=_+()*&^%$€#@!`~×‒„”"
regmatches(x, gregexpr("[[:punct:]]", x, perl=TRUE))  # base R
@


However, the details of the characters' belongingness
to this class depend on the current locale.
Therefore, the reader might obtain different results when calling the above.

\pkg{ICU}, on the other hand, always gives:

<<>>=
stri_extract_all_regex(x, "[[:punct:]]")    # equivalently: \p{P}
@

Here, \code{[:punct:]} is merely a synonym for \code{\textbackslash{}p\{P\}}.
Further, \code{\textbackslash{}p\{S\}} captures symbols:

<<>>=
stri_extract_all_regex(x, "\\p{S}")         # symbols
@


We strongly recommend, wherever possible, the use of the portable
``\code{[\textbackslash{}p\{P\}\textbackslash{}p\{S\}]}''
as an alternative to the \pkg{PCRE}'s ``\code{[:punct:]}''.



\subsection{Alternating and grouping subexpressions}

The alternation operator, ``\code{|}'',
matches either its left or its right branch,
for instance:

<<>>=
x <- "spam, egg, ham, jam, algae, and an amalgam of spam, all al dente"
stri_extract_all_regex(x, "spam|ham")
@

``\code{|}'' has a very low precedence. Therefore, if we wish to
introduce an alternative of subexpressions,
we need to group them, e.g., between round brackets\footnote{Which have
the side-effect of creating new capturing groups, see below for a discussion.}.
For instance, ``\code{(sp|h)am}'' matches either
``\code{spam}'' or ``\code{ham}''.

Also, matching is always done left-to-right, on a first-come, first-served basis.
Hence, if the left branch is a subset of the right one, the latter will
never be matched.
In particular, ``\code{(al|alga|algae)}'' can only match ``\code{al}''.
To fix this, we can write ``\code{(algae|alga|al)}''.


\paragraph{Non-grouping parentheses.}
Some parenthesised subexpressions -- those in which the opening bracket is followed by the question mark -- have a distinct meaning.
In particular, ``\code{(?\#...)}'' denotes a free-format comment
that is ignored by the regex parser:

<<>>=
stri_extract_all_regex(x,
  "(?# match 'sp' or 'h')(sp|h)(?# and 'am')am|(?# or match 'egg')egg")
@

\noindent
Nevertheless, constructing more sophisticated regexes by concatenating
subfragments thereof may sometimes be more readable:

<<>>=
stri_extract_all_regex(x,
  stri_join(
      "(sp|h)",   # match either 'sp' or 'h'
      "am",       # followed by 'am'
    "|",            # ... or ...
      "egg"       # just match 'egg'
))
@


What is more, e.g., ``\code{(?i)}'' enables the \code{case\_insensitive}
mode.

<<>>=
stri_count_regex("Spam spam SPAMITY spAm", "(?i)spam")
@

For more regex flags, we kindly refer the reader to Table~\ref{Tab:regex_opts} again.




\subsection{Quantifiers}

More often than not, a variable number of
instances of the same subexpression needs to be captured
or its presence should be made optional.
This can be achieved by means of the following quantifiers:
\begin{itemize}
\item ``\code{?}'' matches 0 or 1 times;
\item ``\code{*}'' matches 0 or more times;
\item ``\code{+}'' matches 1 or more times;
\item ``\code{\{n,m\}}'' matches between \code{n} and \code{m} times;
\item ``\code{\{n,\}}'' matches at least \code{n} times;
\item ``\code{\{n\}}'' matches exactly \code{n} times.
\end{itemize}
These operators are applied to the preceding atoms.
For example, ``\code{ba+}'' captures
\code{"ba"}, \code{"baa"}, \code{"baaa"}, etc., but not \code{"b"} alone.


By default, the quantifiers are greedy -- they match the
repeated subexpression as many times as possible. The ``\code{?}'' suffix
(hence, quantifiers such as ``\code{??}'', ``\code{*?}'', ``\code{+?}'', and so forth) tries with
as few occurrences as possible (to obtain a match still).


<<>>=
x <- "sp(AM)(maps)(SP)am"
stri_extract_all_regex(x,
  c("\\(.+\\)",    # [[1]] greedy
    "\\(.+?\\)",   # [[2]] lazy
    "\\([^)]+\\)"  # [[3]] greedy (but clever)
))
@

The first regex is greedy: it matches an opening bracket,
then as many characters as possible (including ``\code{)}'')
that are followed by a closing bracket.
The two other patterns terminate as soon as the first closing
bracket is found.



Let us stress that the quantifier is applied to the subexpression
that stands directly before it. Grouping parentheses can be used in case
they are needed.

<<>>=
stri_extract_all_regex("12, 34.5, 678.901234, 37...629, ...",
  c("\\d+\\.\\d+", "\\d+(\\.\\d+)?"))
@

Here, the first regex matches digits, a dot, and another series of digits.
The second one finds digits which are possibly (but not necessarily)
followed by a dot and a digit sequence.





\paragraph{Performance notes.}
\pkg{ICU}, just like \pkg{PCRE}, uses a nondeterministic finite automaton-type
algorithm. Hence, due to backtracking, some ill-defined regexes can lead to
exponential matching times (e.g., ``\code{(a+)+b}'' applied on \code{"aaaa...aaaaac"}).
If such patterns are expected, setting the \code{time\_limit} or \code{stack\_limit}
option is recommended.

<<time-limit, cache=TRUE, error=TRUE>>=
system.time(tryCatch({
  stri_detect_regex("a" %s*% 1000 %s+% "c", "(a+)+b", time_limit=1e5)
}, error=function(e) cat("stopped.")))
@


Nevertheless, oftentimes such regexes can be naturally
reformulated to  fix the underlying issue.
The \pkg{ICU} User Guide on Regular Expressions also recommends using
possessive quantifiers (``\code{?+}'', ``\code{*+}'', ``\code{++}'', and so on),
which match as many times as possible but, contrary to
the plain-greedy ones, never backtrack when they happen to consume too much data.

See also the \pkg{re2r} (a wrapper around the \pkg{RE2} library;
\citealp{re2r}) package's
documentation and the references therein for a discussion.




\subsection{Capture groups and references thereto}\label{Sec:Capturing}

Round-bracketed subexpressions carry one additional
characteristic: they form the so-called capture groups that can
be extracted separately or be referred to in other parts of the same regex.




\paragraph{Extracting capture group matches.}
The above is evident when we use the
versions of \code{stri\_extract()} that are sensitive to the presence
of capture groups:


<<>>=
x <- "name='Sir Launcelot', quest='Seek the Grail', favecolour='blue'"
stri_match_all_regex(x, "(\\w+)='(.+?)'")
@

The findings are presented in a matrix form. The first column
gives the complete matches, the second column stores the matches to the
first capture group, and so forth.


If we just need the grouping part of ``\code{(...)}'', i.e.,
without the capturing feature,
``\code{(?:...)}'' can be applied.
Also, named capture groups defined like
``\code{(?<name>...)}'' are fully supported
since version 1.7.1 of our package (for historical notes see
\citealp{namedCapture}).

<<>>=
stri_match_all_regex(x, "(?:\\w+)='(?<value>.+?)'")
@



\paragraph{Locating capture group matches.}
The \code{capture_groups} attribute in \code{stri_locate_*_regex}
enables us to pinpoint the matches to the parenthesised subexpressions
as well:

<<>>=
stri_locate_all_regex(x, "(?<key>\\w+)='(?<value>.+?)'",
  capture_groups=TRUE, get_length=TRUE)
@

\noindent
Note that each item in the resulting list
is equipped with a \code{"capture_groups"} attribute.
For instance,
\code{attr(result[[1]], "capture_groups")[[2]]}
extracts the locations of the matches to the 2nd capture group
in the first input string.

\paragraph{Replacing with capture group matches.}
Matches to particular capture groups can be recalled in replacement strings
when using \code{stri\_replace()}.
Here, the match in its entirety is denoted with ``\code{\$0}'',  then
``\code{\$1}'' stores whatever was caught by the first capture group,
``\code{\$2}'' is the match to the second capture group, etc.
Moreover, ``\code{\textbackslash{}\$}'' gives the dollar-sign.


<<>>=
stri_replace_all_regex(x, "(\\w+)='(.+?)'", "$2 is a $1")
@

Named capture groups can be referred to too:

<<>>=
stri_replace_all_regex(x, "(?<key>\\w+)='(?<value>.+?)'",
  "${value} is a ${key}")
@

\paragraph{Back-referencing.}
Matches to capture groups can also be part of the regexes themselves.
For example, ``\code{\textbackslash{}1}'' denotes whatever
has been consumed by the first capture group.

Even though, in general, parsing \proglang{HTML} code with regexes is
not recommended, let us consider the following examples:

<<>>=
stri_extract_all_regex("<strong><em>spam</em></strong><code>eggs</code>",
  c("<[a-z]+>.*?</[a-z]+>", "<([a-z]+)>.*?</\\1>"))
@


The second regex guarantees that the match will include all characters
between the opening \code{<tag>} and the corresponding (not: any)
closing \code{</tag>}.
Named capture groups can be referenced using the
\code{\textbackslash{}k<name>} syntax
(the angle brackets are part of the token), as in, e.g.,
``\code{<(?<tagname>[a-z]+)>.*?</\textbackslash{}k<tagname>>}''.





\subsection{Anchoring}

Lastly, let us mention the ways to match a pattern
at a given abstract position within a string.

\paragraph{Matching at the beginning or end of a string.}
``\code{\^{}}'' and ``\code{\$}''  match, respectively,
start and end of the string
(or each line within a string, if the \code{multi_line} option is set to \code{TRUE}).

<<>>=
x <- c("spam egg", "bacon spam", "spam", "egg spam bacon", "sausage")
p <- c("spam", "^spam", "spam$", "spam$|^spam", "^spam$")
structure(outer(x, p, stri_detect_regex), dimnames=list(x, p))
@

The 5 regular expressions match ``\code{spam}'', respectively,
anywhere within the string, at the beginning,
at the end, at the beginning or end,
and in strings that are equal to the pattern itself.



\paragraph{Matching at word boundaries.}
Furthermore, ``\code{\textbackslash{}b}'' matches
at a ``word boundary``, e.g., near spaces, punctuation marks,
or at the start/end of a string (i.e., wherever there is a transition
between a word, ``\code{\textbackslash{}w}'', and a non-word character,
``\code{\textbackslash{}W}'', or vice versa).

In the following example, we match
all stand-alone numbers\footnote{This regular expression
is provided for didactic purposes only.}:


<<>>=
stri_extract_all_regex("12, 34.5, J23, 37.629cm", "\\b\\d+(\\.\\d+)?+\\b")
@

Note the possessive quantifier, ``\code{?+}'':
try matching a dot and a sequence of digits,
and if it is present but not followed by a word boundary,
do not retry by matching a word boundary only.



\paragraph{Looking behind and ahead.}
There are also ways to guarantee that a pattern occurrence
begins or ends with a match to some subexpression:
``\code{(?<=...)...}'' is the so-called  look-behind, whereas
``\code{...(?=...)}'' denotes the look-ahead.
Moreover, ``\code{(?<!...)...}'' and ``\code{...(?!...)}'' are
their negated (``negative look-behind/ahead'') versions.



<<>>=
stri_extract_all_regex("I like spam, spam, eggs, and spam.",
  c("\\w+(?=[,.])", "\\w++(?![,.])"))
@

The first regex captures words that end with ``\code{,}'' or ``\code{.}''.
The second one matches words that end neither with ``\code{,}'' nor ``\code{.}''.






\section{Collation}\label{Sec:collator}



Historically, code-pointwise comparison had been used in most string comparison
activities, especially when strings in ASCII (i.e., English) were
involved. However, nowadays %(in the ``Internet era'')
this does not necessarily constitute the most suitable
approach to the processing of natural-language texts.
In particular, a code-pointwise matching
neither takes accented and conjoined letters nor ignorable punctuation and case
into account.



The \pkg{ICU} Collation Service\footnote{See the \pkg{ICU} User Guide on {Collation},
\url{https://unicode-org.github.io/icu/userguide/collation/}.}
provides the basis for string comparison activities  such as
string sorting and searching, or determining if two strings are equivalent.
This time, though, due to its conformance to
the Unicode Collation Algorithm \citep{uts10:collation},
we may expect that the generated results
will meet the requirements of the culturally correct
natural language processing in any locale.





\subsection{Locales}

String collation is amongst many locale-sensitive operations  available
in \pkg{stringi}. Before proceeding any further, we should
first discuss how we can parameterise the \pkg{ICU} services
so as to deliver the results that reflect the expectations
of a specific user community, such as the speakers of different languages
and their various regional variants.

\paragraph{Specifying locales.}
A locale specifier\footnote{%
Locale specifiers in \pkg{ICU} are platform-independent.
This is not the case for their base-\proglang{R} counterparts, see
\code{help("locales")}, e.g., we have \code{"Polish\_Poland"} on Windows
vs~\code{"pl\_PL"} on Linux.}
is of the form
\code{"Language"}, \code{"Language_Country"}, or \code{"Language_Country_Variant"},
where:
\begin{itemize}
\item
\code{Language} is, most frequently, a two- or three-letter code that conforms to
the ISO-639-1 or ISO-630-2 standard, respectively;
e.g., \code{"en"} or \code{"eng"} for English, \code{"es"} or \code{"spa"}
for Spanish, \code{"zh"} or \code{"zho"} for Chinese, and \code{"mas"} for Masai
(which lacks the corresponding two-letter code);
however, more specific language identifiers may also be
available, e.g., \code{"zh_Hans"} for Simplified-
and \code{"zh_Hant"} for Traditional-Chinese
or \code{"sr_Cyrl"} for Cyrillic- and \code{"sr_Latn"} for Latin-Serbian;


\item
\code{Country} is a two-letter code following the ISO-3166 standard
that enables different language conventions within the same language;
e.g., the US-English (\code{"en_US"}) and Australian-English (\code{"en_AU"})
not only observe some differences in spelling and vocabulary but also
in the units of measurement;

\item
\code{Variant} is an identifier indicating a preference towards
some convention within the same country; e.g., \code{"de_DE_PREEURO"}
formats currency values using the pre-2002 Deutsche Mark (DEM).
\end{itemize}
Moreover,  following the
``\code{@}'' symbol, semicolon-separated ``\code{key=value}'' pairs
can be appended to the locale specifier, in order to
customise some locale-sensitive services even further
(see below for an example using ``\code{@collation=phonebook}''
and Section~\ref{Sec:datetime} for ``\code{@calendar=hebrew}'', amongst others).


{\color{red}

}





\paragraph{Listing locales.}
To list the available locale identifiers, we call \code{stri_locale_list()}.

<<>>=
length(stri_locale_list())
@

\noindent
As the number of supported locales is very high, here we shall
display only 5 randomly chosen ones:

<<echo=-1>>=
set.seed(514678)
sample(stri_locale_list(), 5)
@



\paragraph{Querying for locale-specific services.}
The availability of locale-specific services can only be determined during
the request for a particular resource\footnote{For more details,
see the \pkg{ICU} User Guide on {Locales},
\url{https://unicode-org.github.io/icu/userguide/locale/}.},
which may depend on the \pkg{ICU} library version
actually in use as well as the way the \pkg{ICU} Data Library (\pkg{icudt})
has been packaged. Therefore, for maximum portability,
it is best to rely on the \pkg{ICU} library bundle that
is shipped with \pkg{stringi}.
This is the case on Windows and macOS, whose users typically download the
pre-compiled versions of the package from CRAN.
However, on various flavours of GNU/Linux and other Unix-based systems,
the system \pkg{ICU} is used more eagerly\footnote{
See, e.g., software packages
\code{libicu-dev} on Debian/Ubuntu or \code{libicu-devel} on RHL/Fedora/OpenSUSE.
For more details regarding the configure/build process of \pkg{stringi},
refer to the \code{INSTALL} file.}.
To force building \pkg{ICU} from sources, we may call:

<<eval=FALSE>>=
install.packages("stringi", configure.args="--disable-pkg-config")
@

Overall, if a requested service is unavailable
in a given locale, the best possible match is returned.


\paragraph{Default locale.}
Each locale-sensitive operation in \pkg{stringi} selects the current
default locale if no locale has been explicitly requested,
i.e., when a function's \code{locale} argument (see Table~\ref{Tab:collator_opts})
is left alone in its ``\code{NULL}'' state.
The default locale is initially set to match the system locale on the current
platform, and may be changed with \code{stri_locale_set()}, e.g.,
in the very rare case of improper automatic locale detection.

As we have stated in the introduction, in this paper we use:

<<>>=
stri_locale_get()
@

\noindent
i.e., the Australian-English locale
(which formats dates like ``29 September 2021'' and
uses metric units of measurement).








\subsection{Testing string equivalence}\label{Sec:Equivalence}



In Unicode, some characters may have multiple representations.
For instance, ``LATIN SMALL LETTER A WITH OGONEK'' (``ą'') can be stored
as a single code point U+0105 or as a sequence
that is comprised of the letter ``LATIN SMALL LETTER A'', U+0061, and
the ``{COMBINING OGONEK}'', U+0328 (when rendered properly, they
should appear as if they were identical glyphs).
This is an example of canonical equivalence of strings.

Testing for the Unicode equivalence between strings
can be performed by calling \code{\%s==\%} and, more generally,
\code{stri\_cmp\_equiv()}, or their negated versions,
\code{\%s!=\%} and \code{stri\_cmp\_nequiv()}.

In the example below we have: a followed by ogonek (two code points)
vs a with ogonek (single code point).

<<>>=
"a\u0328" %s==% "ą"             # a, ogonek == a with ogonek
@



There are also functions for removing and indicating
duplicated elements in a character vector:

<<>>=
x <- c("Gągolewski", "Gagolewski", "Ga\u0328golewski")
stri_unique(x)
stri_duplicated(x)  # from_last=FALSE
@

Moreover, \code{stri_duplicated_any()}
returns the index of the first non-unique element.






\subsection{Linear ordering of strings}



Operators such as \code{\%s<\%}, \code{\%<=\%}, etc.,
and the corresponding functions
\code{stri_cmp_lt()} (``less than''),
\code{stri_cmp_le()} (``less than or equal''), etc.,
implement locale-sensitive linear orderings of strings.
Moreover, \code{stri_sort()} returns the lexicographically-sorted
version of a given input vector, \code{stri_order()} yields
the corresponding (stable) ordering permutation,
and \code{stri_rank()} ranks strings within a vector.



For instance, here is a comparison in the current default locale
(Australian-English):

<<>>=
"chaotic" %s<% "hard"  # c < h
@

Similar comparison in Polish:

<<>>=
stri_cmp_lt("chłodny", "hardy", locale="pl_PL")  # c < h
@

And now for something completely different -- the Slovak language:

<<>>=
stri_cmp_lt("chladný", "hladný", locale="sk_SK") # ch > h
@



This is an example of the locale-aware comparison
that is context-sensitive
and which goes beyond the simple code-pointwise comparison.
In the example above, a contraction occurred:
in Slovak,
two code points ``\code{ch}'' are treated as a single entity
and are sorted after ``\code{h}'':


Compare the ordering of Polish and Slovak words:

<<>>=
stri_sort(c("chłodny", "hardy", "cichy", "cenny"), locale="pl_PL")
stri_sort(c("cudný", "chladný", "hladný", "čudný"), locale="sk_SK")
@



An opposite situation is called an expansion:

<<>>=
german_k_words <- c("können", "kondensieren", "kochen", "korrelieren")
stri_sort(german_k_words, locale="de_DE")
stri_sort(german_k_words, locale="de_DE@collation=phonebook")
@


In the latter example, where we used the German phone-book order,
\code{"ö"} is treated as \code{"oe"}.






\subsection{Collator options}\label{Sec:collator_opts}

Table~\ref{Tab:collator_opts} lists the options
that can be passed to \code{stri_opts_collator()} via the dot-dot-dot
parameter, ``\code{...}'', in
all the functions that rely on the \pkg{ICU} Collator.
Below we would like to attract the kind reader's attention
to some of them.




\begin{table}[bt!]
\centering

\begin{tabularx}{1.0\linewidth}{lX}
\toprule
\bfseries{Option}            &\bfseries Purpose \\
\midrule
\code{locale}             & a string specifying the locale to use; \code{NULL}
(default) or \code{""} for the current default locale as indicated by
\code{stri\_locale\_get()} \\
\midrule
\code{strength}           & an integer in $\{1,2,3,4\}$ defining collation strength;
1 for the most permissive collation rules, 4 for the strictest ones;
defaults to 3 \\
\midrule
\code{uppercase\_first}    & logical; \code{NA} (default) orders upper
and lower
case letters in accordance to their tertiary weights, \code{TRUE} forces upper
case letters to sort before lower case letters, \code{FALSE} does the opposite \\
\midrule
\code{numeric}            & logical; if \code{TRUE}, a collation key
for the numeric value of substrings of digits is generated; this is a way to
make \code{"100"} ordered
after \code{"2"}; defaults to \code{FALSE} \\
\midrule
\code{case\_level}         & logical; if \code{TRUE}, an extra case level
(positioned before the third level) is generated; defaults to \code{FALSE} \\
\midrule
\code{normalisation}      & logical; if \code{TRUE}, then an incremental
check is performed to see whether input data are in the FCD (``fast C or D'') form;
if data are not in the FCD form, the incremental NFD normalisation is performed,
see Section~\ref{Sec:normalisation}; defaults to \code{FALSE}   \\
\midrule
\code{alternate\_shifted}  & logical; if \code{FALSE} (default),
all code points with non-ignorable primary weights are handled in the same way;
\code{TRUE} causes the code points
with primary weights that are less than or equal to the variable top value
to be ignored on the primary level and moved to the quaternary level; this can be
used to, e.g., ignore punctuation, see the examples provided \\ \midrule
\code{french}             & logical; \code{TRUE} results in secondary
weights being considered backwards, i.e., ordering according to the last accent difference
-- nowadays only used in Canadian-French; defaults to \code{FALSE} \\
\bottomrule
\end{tabularx}

\caption{\label{Tab:collator_opts} Options for the \pkg{ICU}
Collator that can be passed to \code{stri\_opts\_collator()}.}
\end{table}




\paragraph{Collation strength.}
The Unicode Collation Algorithm \citep{uts10:collation}
can go beyond simple canonical equivalence:
it can treat some other (depending on the context)
differences as negligible too.

The \code{strength} option controls the Collator's ``attention to detail''.
For instance, it can be used to make the ligature ``ff'' (U+FB00)
compare equal to the two-letter sequence ``f{}f'':

<<>>=
stri_cmp_equiv("\ufb00", "ff", strength=2)
@

\noindent
which is not the case at the default strength level (3).




Generally, four (nested) levels of inter-string differences can be distinguished:
\begin{enumerate}
\item A primary difference -- the strongest one -- occurs where
there is a mismatch between base characters (e.g., \code{"a"} vs~\code{"b"}).

\item Some character accents can be considered a secondary difference
in many languages. However, in other ones, an accented letter is considered
a unique letter.

\item Distinguishing between upper- and lower case typically happens
on the tertiary level (see, however, the \texttt{case\_level} option).


\item If \code{alternate\_shifted} is \code{TRUE},
differences in punctuation
can be determined at the quaternary level. This is also meaningful
in the processing of Hiragana text.


\end{enumerate}

\vbox{
\paragraph{Ignoring case.}
Note which strings are deemed equivalent when
considering different collation strengths:

<<>>=
x <- c("gro\u00df", "gross", "GROSS", "Gro\u00df", "Gross")
stri_unique(x, strength=1)                  # ß == ss, case insensitive
stri_unique(x, strength=2)                  # ß != ss, case insensitive
@
}


Hence, strength equal to 1 takes only primary differences
into account. Strength of 2 will also be sensitive to secondary differences
(distinguishes between ``ß'' and ``ss'' above), but will ignore
tertiary differences (case).

Also, introducing an extra case level yields
a case sensitive comparison that ignores secondary differences:

<<>>=
stri_unique(x, strength=1, case_level=TRUE) # ß == ss, case sensitive
@


\paragraph{Ignoring some punctuation.}
Here are some effects of changing the  \code{alternate\_shifted} option,
which allows for ignoring some punctuation marks:

<<>>=
x <- c("code point", "code-point", "codepoint", "CODE POINT", "CodePoint")
stri_unique(x, alternate_shifted=TRUE)  # strength=3
stri_unique(x, alternate_shifted=TRUE, strength=2)
@

In the latter case, all strings are considered equivalent.
Ignoring case but not punctuation yields:

<<>>=
stri_unique(x, strength=2)
@





\paragraph{Backward secondary sorting.}
The French Canadian Sorting Standard CAN/CSA Z243.4.1 (historically this had been
the default for all French locales) requires the word ordering with respect
to  the last accent difference. Such a behaviour can be applied
either by setting the French-Canadian locale or by passing the \code{french=TRUE}
option to the Collator.

<<>>=
stri_sort(c("cote", "côte", "coté", "côté"), locale="fr_FR")
stri_sort(c("cote", "côte", "coté", "côté"), locale="fr_CA") # french=TRUE
@



\paragraph{Sorting numerals.}
By default, just like in base \proglang{R} and most other programming
languages, a lexicographic ordering is used: the corresponding
code points are being compared one by one, from left to right,
and once a difference
is detected, the result is returned immediately.

<<>>=
x <- c("a1", "a2", "a11", "a1", "a99", "a10", "a100", "a2", "a9", "a2")
stri_sort(x)
@

For example, \code{"a99"} is ordered after \code{"a100"},
because \code{"a" == "a"} (first characters are equal)
but then \code{"9" > "1"} (second characters are already different).

Let us, however, note the effect of setting the \code{numeric} option
on the sorting of strings that involves numbers:

<<>>=
stri_sort(x, numeric=TRUE)
@




Here is an example of sorting a data frame
with respect to two criteria:

<<echo=-1>>=
set.seed(123)
X <- data.frame(a=x, b=runif(length(x)))
X[order(-stri_rank(X$a, numeric=TRUE), X$b), ]
@

The object is now ordered by the first column decreasingly
(using a ``numeric'' order) and ties are resolved based on increasing
values in the second column.


\paragraph{A note on compatibility equivalence.}
In Section~\ref{Sec:normalisation} we describe different ways to normalise
canonically equivalent code point sequences so that they are
represented by the same code points, which can account for some negligible
differences (as in the ``a with ogonek'' example above).

Apart from ignoring punctuation and case, the Unicode Standard Annex \#15 \citep{usa15:normalization}
also discusses the so-called compatibility equivalence of strings.
This is a looser form of similarity; it is observed where
there is the same abstract content, yet displayed
by means of different glyphs, for instance ``¼'' (U+00BC) vs~``\code{1/4}''
or ``$\mathbb{R}$'' vs~``\proglang{R}''.
In the latter case, whether these should be treated as equal,
depends on the context (e.g., this can be the set of real numbers
vs~one's favourite programming language).
Compatibility decompositions (NFKC, NFKD)
mentioned in Section~\ref{Sec:normalisation}
or other types of transliteration can be used to normalise strings so that
such differences are not accounted for.

Also, for ``fuzzy'' matching of strings,
the \pkg{stringdist} package \citep{stringdist} might be helpful.













\subsection{Searching for fixed strings revisited}

The \pkg{ICU} Collator can also be utilised
where there is a need to locate the occurrences of simple textual patterns.
The counterparts of the string search functions
described in Section~\ref{Sec:fixed}
have their names ending with \code{*_coll()}.
They are slower than them, but are more appropriate in NLP activities.

For instance:

<<>>=
stri_detect_coll("Er ist so groß.", "GROSS", strength=1, locale="de_AT")
stri_detect_coll("On je chladný", "chladny", strength=1, locale="sk_SK")
@





\section{Other operations}\label{Sec:other}

In the sequel, we cover the functions that deal with
text boundary detection, random string generation,
date/time formatting and parsing, amongst others.


\subsection{Analysing text boundaries}\label{Sec:BoundaryAnalysis}

Text boundary analysis aims at locating linguistic delimiters
for the purpose of splitting text into lines,
word-wrapping, counting characters or
words, locating particular text units (e.g.,
the 3rd sentence), etc.

Generally, text boundary analysis is a locale-sensitive operation,
see \citep{usa29:segmentation}.
For example, in Japanese and Chinese, spaces are not used for
separation of words  --
a line break can occur even in the middle of a word. Nevertheless,
these languages have punctuation and diacritical marks that cannot
start or end a line, so this must also be taken into account.

The \pkg{ICU} Break Iterator\footnote{See the \pkg{ICU} User Guide
on {Boundary Analysis}, \url{https://unicode-org.github.io/icu/userguide/boundaryanalysis/}.}
comes in four flavours (see the \code{type} option
in \code{stri_opts_brkiter()}):
\code{character}, \code{work}, \code{line_break}, and \code{sentence}.

We have access to functions such as
\code{stri_count_boundaries()},
\code{stri_split_boundaries()},
\code{stri_extract_*_boundaries()}, and
\code{stri_locate_*_boundaries()},
as well as their specialised
versions:
\code{stri_count_words()},
\code{stri_extract_*_words()}, and
\code{stri_split_lines()}, amongst others.
For example:


<<>>=
x <- "The\u00a0above-mentioned    features are useful. " %s+%
  "My hovercraft is full of eels, eggs, and spam."
@

Number of sentences:

<<>>=
stri_count_boundaries(x, type="sentence")
@

The list of all the words:

<<>>=
stri_extract_all_words(x)
@





\subsection{Trimming, padding, and other formatting}

The following functions can be used for pretty-printing
character strings or text on the console,
dynamically generating reports (e.g.,
with \code{Sweave()} or \pkg{knitr}; see \citealp{knitr}),
or creating text files
(e.g., with \code{stri\_write\_lines()}; see Section~\ref{Sec:read_lines}).



\paragraph{Padding.}
\code{stri\_pad()} pads strings with some character so that they
reach the desired widths (as in \code{stri_width()}).
This can be used to centre, left-, or right-align a message
when printed with, e.g.,  \code{cat()}.


<<>>=
cat(stri_pad("SPAMITY SPAM", width=77, side="both", pad="."))
@







\paragraph{Trimming.}
A dual operation is that of trimming from the left or right side
of strings:

<<>>=
x <- "      spam, eggs, and lovely spam.\n"
stri_trim(x)  # side="both"
@



\paragraph{Word wrapping.}
The \code{stri\_wrap()} function splits each (possibly long)
string in a character vector into chunks of at most a given width.
By default, the dynamic word wrap algorithm \citep{Knuth:wrap}
that minimises the raggedness of the formatted text is used.
However, there is also an option (\code{cost\_exponent=0})
to use the greedy alignment,
for compatibility with the built-in \code{strwrap()}.

<<echo=-1>>=
set.seed(1233)
x <- stri_rand_lipsum(1)  # random text paragraph
cat(stri_wrap(x, width=74, indent=8, exdent=4, prefix="> "), sep="\n")
@







\paragraph{Applying string templates.}
\code{stri_sprintf()} is a Unicode-aware rewrite of the built-in
\code{sprintf()} function. In particular, it enables formatting and
padding based on character width, not just the number of code points.
The function is also available as a binary operator \code{\%s\$\%},
which is similar to \proglang{Python}'s \code{\%} overloaded for
objects of type \code{str}.

<<>>=
cat(stri_sprintf("[%6s]", c("abcd", "\u200b\u200b\u200bąß²€")), sep="\n")
@

The above guarantees that the two output strings are
of at least width of 6 (plus the square brackets).



\subsection{Generating random strings}

Apart from \code{stri\_rand\_lipsum()},
which produces random-ish text paragraphs (``placeholders'' for real text),
we have access to a function that generates sequences of characters
uniformly sampled (with replacement) from a given set.

For example, here are 5 random ACTG strings of lengths from 2 to 6:

<<echo=-1>>=
set.seed(123)
stri_rand_strings(5, 2:6, "[ACTG]")
@



See Section~\ref{Sec:RegexIndividualChars}
and \code{help("stringi-search-charclass")}
for different ways to specify character sets.





\subsection{Transliterating}

Transliteration, in its broad sense, deals with the substitution
of characters or their groups for different ones, according to some
well-defined, possibly context-aware, rules.
It may be useful, amongst others, when "normalising"
pieces of strings or identifiers so that they can be more easily
compared with each other.


\paragraph{Case mapping.}
Mapping to  upper, lower, or title case
is a language- and context-sensitive operation
that can change the total number of code points in a string.



<<>>=
stri_trans_toupper("groß")
stri_trans_tolower("Iİ", locale="tr_TR")               # Turkish
stri_trans_totitle("ijsvrij yoghurt", locale="nl_NL")  # Dutch
@







\paragraph{Mapping between specific characters.}
When a fast 1-to-1 code point translation is required, we can call:

<<>>=
stri_trans_char("GATAAATCTGGTCTTATTTCC", "ACGT", "tgca")
@

Here, ``\code{A}'', ``\code{C}'', ``\code{G}'', and ``\code{T}''
is replaced with
``\code{t}'', ``\code{g}'', ``\code{c}'', and ``\code{a}'', respectively.



\paragraph{General transforms.}
\code{stri_trans_general()} provides access to a wide range of text transforms
defined by \pkg{ICU}, whose catalogue can be accessed by calling
\code{stri_trans_list()}.

<<echo=-1>>=
set.seed(12345)
sample(stri_trans_list(), 9)  # a few random entries
@

For example, below we apply a transliteration chain:
first, we convert to upper case, and then we convert characters
in the Latin script to ASCII.


<<>>=
stri_trans_general("groß© żółć La Niña köszönöm", "upper; latin-ascii")
@


Custom rule-based transliteration is also
supported\footnote{See the \pkg{ICU} User Guide on {General Transforms}
for more details,
\url{https://unicode-org.github.io/icu/userguide/transforms/general/}.
}. It can be used, for instance, to generate different romanisations
of non-Latin alphabets.















\subsection{Parsing and formatting date and time}\label{Sec:datetime}

In base \proglang{R}, dealing with temporal data in regional
settings other than the current locale is somewhat difficult.
For instance, many will find the task of parsing the following Polish date problematic:

<<>>=
x <- "28 września 2021 r., godz. 17:17:32"
@

\pkg{stringi} connects to the \pkg{ICU} date and time
services so that this becomes an easy exercise:


<<>>=
stri_datetime_parse(x, "dd MMMM yyyy 'r., godz.' HH:mm:ss",
  locale="pl_PL", tz="Europe/Warsaw")
@

This function returns an object of class \code{POSIXct},
for compatibility with base \proglang{R}.
Note, however, that
\pkg{ICU} uses its own format patterns\footnote{
See the \pkg{ICU} User Guide on {Formatting Dates and Times},
\url{https://unicode-org.github.io/icu/userguide/format_parse/datetime/}.
}. For convenience,
\code{strftime()}- and \code{strptime()}-like
templates can be converted with \code{stri_datetime_fstr()}:

<<>>=
stri_datetime_parse(x,
  stri_datetime_fstr("%d %B %Y r., godz. %H:%M:%S"),
  locale="pl_PL", tz="Europe/Warsaw")
@



For example, here is how we can access different calendars:

<<>>=
stri_datetime_format(
  stri_datetime_create(2020, 1:12, 1),
  "date_long",
  locale="@calendar=hebrew")
stri_datetime_format(
  stri_datetime_create(2020, c(2, 8), c(4, 7)),
  "date_full",
  locale="ja_JP@calendar=japanese")
@

\noindent
Above we have selected the Hebrew calendar within the
English locale and the Japanese calendar in the Japanese locale.












\section{Input and output}\label{Sec:io}

This section deals with some more advanced topics related to the
operability of text processing applications
between different platforms. In particular, we discuss
how to assure that data read from various input connections
are interpreted in the correct manner.



\subsection{Dealing with Unicode code points}\label{Sec:codepoints}

The Unicode Standard (as well as the
Universal Coded Character Set, i.e., ISO/IEC 10646)
currently defines over 140{,}000 abstract characters together with
their corresponding code points -- integers
between 0 and 1{,}114{,}111 (or 0000${}_{16}$ and 10FFFF${}_{16}$
in hexadecimal notation, see \url{https://www.unicode.org/charts/}).
In particular, here is the number of the code points in
some popular categories (compare Section~\ref{Sec:RegexIndividualChars}),
such as letters, numbers, and the like.

<<>>=
z <- c("\\p{L}", "\\p{Ll}", "\\p{Lu}", "\\p{N}", "\\p{P}", "\\p{S}",
  "\\w", "\\d", "\\s")
structure(stri_count_regex(stri_enc_fromutf32(
  setdiff(1:0x10ffff, c(0xd800:0xf8ff))), z), names=z)
@

Yet, most of the code points are still unallocated.
The Unicode standard is occasionally updated, e.g., the
most recent versions were supplemented with over 1{,}000 emojis.


The first 255 code points are identical to the ones defined
by ISO/IEC 8859-1 (ISO Latin-1;
``Western European''), which itself extends US-ASCII (codes $\le 127=\text{7F}{}_{16}$).
For instance, the code point that we are used to denoting as U+007A
(the ``U+'' prefix is followed by a sequence of hexadecimal digits;
7A${}_{16}$ corresponds to decimal 122) encodes the lower case letter ``z''.
To input such a code point in \proglang{R}, we write:

<<>>=
"\u007A"  # or "\U0000007A"
@


For communicating with \pkg{ICU} and other libraries,
we may need to escape a given string, for example, as follows
(recall that to input a backslash in \proglang{R},
we must escape it with another backslash).

<<>>=
x <- "zß你好"
stri_escape_unicode(x)
@



\bigskip
It is worth noting that despite the fact that some output devices
might be unable to display certain code points correctly
(due to, e.g., missing fonts), the correctness of their
processing with \pkg{stringi} is still guaranteed by \pkg{ICU}.






\subsection{Character encodings}\label{Sec:encoding}

When storing strings in RAM or on the disk,
we need to decide upon the actual way
of representing the code points as sequences of bytes.
The two most popular encodings in the Unicode family are
UTF-8 and UTF-16:

<<>>=
x <- "abz0ąß你好!"
stri_encode(x, to="UTF-8", to_raw=TRUE)[[1]]
stri_encode(x, to="UTF-16LE", to_raw=TRUE)[[1]]  # little-endian
@

\proglang{R}'s current platform-default encoding, which we shall
refer to as the native encoding, is defined via the
\code{LC_CTYPE} locale category in
\code{Sys.getlocale()}. This is the representation assumed,
e.g., when reading data from the standard input
or from files (e.g., when \code{scan()} is called).
For instance, Central European versions of Windows will assume
the ``\code{windows-1250}'' code page.
MacOS as well as most Linux boxes work with UTF-8 by default\footnote{
It is expected that future \proglang{R} releases will support UTF-8 natively
thanks to the Universal \proglang{C} Runtime (UCRT) that is available for Windows 10.}.

All strings in \proglang{R} have an associated encoding mark
which can be read by calling \code{Encoding()} or, more conveniently,
\code{stri\_enc\_mark()}.
Most importantly, strings in ASCII, ISO-8859-1 (``\code{latin1}''),
UTF-8, and the native encoding can coexist.
Whenever a non-Unicode string is passed to a function in \pkg{stringi},
it is silently converted to UTF-8 or UTF-16, depending on the requested
operation (some \pkg{ICU} services are only available for {UTF-16} data).
Over the years, this has proven a robust, efficient, and maximally portable
design choice -- Unicode can be thought of as a superset of every other encoding.
Moreover, in order to guarantee the correctness and high performance of
the string processing pipelines, \pkg{stringi} always\footnote{With a few
obvious exceptions, such as \code{stri\_encode()}.} outputs
UTF-8 data.


\subsection{Reading and writing text files and converting between encodings}\label{Sec:read_lines}

According to a report by W3Techs\footnote{See
\url{https://w3techs.com/technologies/cross/character_encoding/ranking}.},
as of 2021--09--28, 97.3\% of websites use UTF-8.
Nevertheless, other encodings can still be encountered.

\paragraph{Reading and writing text files.}
If we know the encoding of a text file  in advance,
\code{stri\_read\_lines()} can be used to read
the data in a manner similar to the built-in \code{readLines()} function
(but with a much easier access to encoding conversion):

For instance, below we read a text file encoded in ISO-8859-1:

<<>>=
x <- stri_read_lines("ES_latin1.txt", encoding="ISO-8859-1")
head(x, 4)  # x is in UTF-8 now
@


We can call \code{stri_write_lines()} to write the contents
of a character vector to a file (each string will
constitute a separate text line), with any output encoding.




\paragraph{Detecting encoding.}
However, if a file's encoding is not known in advance, there are
a certain functions that can aid in encoding detection.
First, we can read the resource in the form of a raw-type vector:


<<>>=
x <- stri_read_raw("ES_latin1.txt")
head(x, 24)  # vector of type raw
@

Then, to guess the encoding, we can call, e.g.:

<<>>=
stri_enc_isascii(x)
stri_enc_isutf8(x)   # false positives are possible
@

\noindent
Alternatively, we can use:

<<>>=
stri_enc_detect(x)  # based on heuristics
@

\noindent
Nevertheless, encoding detection is an operation that relies on heuristics,
therefore there is a  chance that the output might be imprecise or even
misleading.




\paragraph{Converting encodings.}
Knowing the desired source and destination encoding precisely,
\code{stri_encode()} can be called to perform the conversion.
Contrary to the built-in \code{iconv()}, which relies
on different underlying libraries, the current function is portable
across operating systems.


<<>>=
y <- stri_encode(x, from="ISO-8859-1", to="UTF-8")
@

\code{stri_enc_list()}  provides a list of
supported encodings and their aliases in many different forms.
Encoding specifiers are normalised automatically, e.g.,
\code{"utf8"} is a synonym for \code{"UTF-8"}.


Splitting the output  into text lines gives:

<<>>=
tail(stri_split_lines1(y), 4)  # spoiler alert!
@




















\subsection{Normalising strings}\label{Sec:normalisation}

In Section~\ref{Sec:Equivalence} we have provided some examples
of canonically equivalent strings whose code point representation was different.
Unicode normalisation forms C (Canonical composition, NFC) and D
(Canonical decomposition, NFD) can be applied so that they
will compare equal using bytewise matching \citep{usa15:normalization}.

<<>>=
x <- "a\u0328 ą"   # a, combining ogonek, space, a with ogonek
stri_enc_toutf32(  # code points as decimals
  c(x, stri_trans_nfc(x), stri_trans_nfd(x)))
@

Above we see some example code points before, after NFC,
and after NFD normalisation, respectively.

It might be a good idea to always normalise all the strings
read from external sources (files, URLs) with NFC.

Compatibility composition and decomposition normalisation forms (NFKC and
NFKD, respectively) are also available if
the removal of the formatting distinctions (font variants,
subscripts, superscripts, etc.) is desired. For example:


<<>>=
stri_trans_nfkd("r²︷")
@

Such text might be much easier to process or analyse statistically.



\section{Closing remarks}\label{Sec:conclusions}




We have reviewed the key design principles and facilities available in
\pkg{stringi}, including numerous operations
that  help implement and optimise data processing pipelines.
Let us again stress that many package features are provided by \pkg{ICU}, which
is a platform-independent project. Hence, information
presented above might be of interest to statisticians and data scientists
employing different programming environments as well.



Users seeking Unicode-aware replacements for base \proglang{R}
string processing functions are kindly referred to the \pkg{stringx}
package \citep{stringx}, which is a set of wrappers around \pkg{stringi}
offering a more classic API
(functions such as \code{grepl()}, \code{substring()}, etc.,
compare Table~\ref{Tab:oldstringr}).


\pkg{stringi} functions can also be accessed from within \proglang{C++}
code. Authors of statistical/data analysis software who would like to
speed up their projects are encouraged to check out
the \pkg{ExampleRcppStringi} package available at
\url{https://github.com/gagolews/ExampleRcppStringi},
which serves as a working prototype developed using \pkg{Rcpp} \citep{rcppbook}.














\paragraph{Future of \pkg{stringi}.}
Over the years, many useful \proglang{R} packages related
to text processing have been developed, see \citep{textminingr,textr}
for some reviews.
Several of them are listed in the CRAN Task View
on Natural Language Processing\footnote{
See \url{%
https://cran.r-project.org/web/views/NaturalLanguageProcessing.html}.
}.
At the time of the writing of this paper,
\pkg{stringi} itself had over 200 strong (direct) reverse dependencies
and has established itself as one of the most frequently downloaded
\proglang{R} extension. Its user base is growing steadily.

Most importantly, the package can be relied upon by other software projects
as its API is considered stable and most changes are backward compatible.

Future work will involve the porting of \pkg{stringi} to different
scientific/statistical computing environments, including
\proglang{Julia} and \proglang{Python}
with the \pkg{NumPy} \citep{numpy} ecosystem,
offering more Unicode-aware alternatives to the vectorised text processing
facilities from \code{numpy.char} and \pkg{pandas} \citep[Chap.~7]{pandas}.

Moreover, further extensions of \pkg{stringi} shall be conveyed
in order to provide an even broader coverage of \pkg{ICU} services.






\section*{Acknowledgements}

\pkg{stringi} is an open source project
distributed under the terms of the BSD-3-clause license.
Its most recent development snapshot is available through GitHub at
\url{https://github.com/gagolews/stringi}. The bug- and feature request
tracker can be accessed from
\url{https://github.com/gagolews/stringi/issues}.
Moreover, its homepage -- which includes a reference manual
documents the package's API in detail --
is located at \url{https://stringi.gagolewski.com/}.

The author wishes to thank Hadley Wickham for coming
up with the original \pkg{stringr} package API (see Table~\ref{Tab:oldstringr}).
Also, greatly appreciated are the contributions of all who have donated their
time and effort (in all the possible forms: code, feature suggestions,
ideas, criticism, testing) to make \pkg{stringi} better --
Bartek Tartanus,
Kenneth Benoit,
Marcin Bujarski,
Bill Denney,
Katrin Leinweber,
Jeroen Ooms,
Davis Vaughan,
and many others,
see \url{https://github.com/gagolews/stringi/graphs/contributors}.
More contributions are always welcome.






\bibliography{bibliography}







\begin{table}[p!]
\centering

\begin{tabularx}{1.0\linewidth}{p{3.4cm}p{3.7cm}X}
\toprule
\pkg{stringr} 0.6.2                            & \bfseries Base \proglang{R} 4.1           & \bfseries Purpose \\
\midrule
\code{str\_c()}                                & \code{paste()}, \code{paste0()}, also \code{sprintf()}                          & join strings \\                                                          \midrule
\code{str\_count()}                            & \code{gregexpr()}                      & count pattern matches\\                                                  \midrule
\code{str\_detect()}                           & \code{grepl()}                         & detect pattern matches \\                                                \midrule
\code{str\_dup()}                              & \code{strrep()}                        & duplicate strings\\                                                      \midrule
\code{str\_extract()}, \code{str\_extract\_all()} & \code{regmatches()} with \code{regexpr()}, \code{gregexpr()}                                      & extract (first, all) pattern matches  \\                                 \midrule
\code{str\_length()}                           & \code{nchar()}                         & compute string length \\                                                 \midrule
                                               & \code{nchar(type="width")}             & compute string width \\                                                 \midrule
\code{str\_locate()}, \code{str\_locate\_all()}& \code{regexpr()}, \code{gregexpr()}    & locate (first, all) pattern matches \\                                   \midrule
\code{str\_match()}, \code{str\_match\_all()}  & \code{regmatches()} with \code{regexec()}, \code{gregexec()}    & extract (first, all) matches to regex capture groups \\                  \midrule
\code{str\_pad()}                              &                                        & add whitespaces at beginning or end\\                                    \midrule
\code{str\_replace()}, \code{str\_replace\_all()}   & \code{sub()}, \code{gsub()}       & replace (first, all) pattern matches with a replacement string\\        \midrule
\code{str\_split()}, \code{str\_split\_fixed()}& \code{strsplit()}                      & split up a string into  pieces \\                       \midrule
\code{str\_sub()}    & \code{substr()}, \code{substring()}                     & extract or replace substrings\\                                         \midrule
\code{str\_trim()}                             & \code{trimws()}                        & remove whitespaces from beginning or end \\                              \midrule
\code{str\_wrap()}                             & \code{strwrap()}                       & split strings into text lines \\                                         \midrule
\code{word()}                                  &                                        & extract words from a sentence \\  \midrule
                                               & \code{startsWith()}, \code{endsWith()} & determine if strings start or end with a pattern match \\ \midrule
                                               & \code{tolower()}, \code{toupper()}     & case mapping and folding \\ \midrule
                                               & \code{chartr()}                        & transliteration \\ \midrule
                                               & \code{sprintf()}                       & string formatting \\ \midrule
                                               & \code{strftime()}, \code{strptime()}   & date-time formatting and parsing \\
\bottomrule
\end{tabularx}

\caption{\label{Tab:oldstringr} Functions in (the historical)
\pkg{stringr} 0.6.2 and their counterparts in base \proglang{R} 4.1.}
\end{table}


\end{document}
