// Copyright (C) 2024 Quickwit, Inc.
//
// Quickwit is offered under the AGPL v3.0 and as commercial software.
// For commercial licensing, contact us at hello@quickwit.io.
//
// AGPL:
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

syntax = "proto3";

package quickwit.search;

service SearchService {
  // Root search API.
  // This RPC identifies the set of splits on which the query should run on,
  // and dispatch the several calls to `LeafSearch`.
  //
  // It is also in charge of merging back the results.
  rpc RootSearch(SearchRequest) returns (SearchResponse);

  // Perform a leaf search on a given set of splits.
  //
  // It is like a regular search except that:
  // - the node should perform the search locally instead of dispatching
  //   it to other nodes.
  // - it should be applied on the given subset of splits
  // - Hit content is not fetched, and we instead return so called `PartialHit`.
  rpc LeafSearch(LeafSearchRequest) returns (LeafSearchResponse);

  /// Fetches the documents contents from the document store.
  /// This methods takes `PartialHit`s and returns `Hit`s.
  rpc FetchDocs(FetchDocsRequest) returns (FetchDocsResponse);

  // Perform a leaf stream on a given set of splits.
  rpc LeafSearchStream(LeafSearchStreamRequest) returns (stream LeafSearchStreamResponse);

  // Root list terms API.
  // This RPC identifies the set of splits on which the query should run on,
  // and dispatches the several calls to `LeafListTerms`.
  //
  // It is also in charge of merging back the results.
  rpc RootListTerms(ListTermsRequest) returns (ListTermsResponse);

  // Performs a leaf list terms on a given set of splits.
  //
  // It is like a regular list term except that:
  // - the node should perform the listing locally instead of dispatching
  //   it to other nodes.
  // - it should be applied on the given subset of splits
  rpc LeafListTerms(LeafListTermsRequest) returns (LeafListTermsResponse);

  // Performs a scroll request.
  rpc Scroll(ScrollRequest) returns (SearchResponse);

  // gRPC request used to store a key in the local storage of the targeted node.
  // This RPC is used in the mini distributed immutable KV store embedded in quickwit.
  rpc PutKV(PutKVRequest) returns (PutKVResponse);

  // Gets a key from the local storage of the targeted node.
  // This RPC is used in the mini distributed immutable KV store embedded in quickwit.
  rpc GetKV(GetKVRequest) returns (GetKVResponse);

  rpc ReportSplits(ReportSplitsRequest) returns (ReportSplitsResponse);

  rpc ListFields(ListFieldsRequest) returns (ListFieldsResponse);

  rpc LeafListFields(LeafListFieldsRequest) returns (ListFieldsResponse);

  // Describe how a search would be processed.
  rpc SearchPlan(SearchRequest) returns (SearchPlanResponse);
}

/// Scroll Request
message ScrollRequest {
  /// The `scroll_id` is the given in the response of a search request including a scroll.
  string scroll_id = 1;
  optional uint32 scroll_ttl_secs = 2;
}

message PutKVRequest {
  bytes key = 1;
  bytes payload = 2;
  uint32 ttl_secs = 3;
}

message PutKVResponse {}

message GetKVRequest {
  bytes key = 1;
}

message GetKVResponse {
  optional bytes payload = 1;
}


message ReportSplit {
  // Split id (ULID format `01HAV29D4XY3D462FS3D8K5Q2H`)
  string split_id = 2;
  // The storage uri. This URI does NOT include the split id.
  string storage_uri = 1;
}

message ReportSplitsRequest {
  repeated ReportSplit report_splits = 1;
}

message ReportSplitsResponse {}

// -- ListFields -------------------

message ListFieldsRequest {
  // Index ID patterns
  repeated string index_id_patterns = 1;
  // Optional limit query to a list of fields
  // Wildcard expressions are supported.
  repeated string fields = 2;

  // Time filter, expressed in seconds since epoch.
  // That filter is to be interpreted as the semi-open interval:
  // [start_timestamp, end_timestamp).
  optional int64 start_timestamp = 3;
  optional int64 end_timestamp = 4;

  // Control if the the request will fail if split_ids contains a split that does not exist.
  // optional bool fail_on_missing_index = 6;
}

message LeafListFieldsRequest {
  // The index id
  string index_id = 1;
  // The index uri
  string index_uri = 2;
  // Index split ids to apply the query on.
  // This ids are resolved from the index_uri defined in the search_request.
  repeated SplitIdAndFooterOffsets split_offsets = 3;

  // Optional limit query to a list of fields
  // Wildcard expressions are supported.
  repeated string fields = 4;

}

message ListFieldsResponse {
  repeated ListFieldsEntryResponse fields = 1;
}

message ListFieldsEntryResponse {
  string field_name = 1;
  ListFieldType field_type = 2;
  // The index ids the field exists
  repeated string index_ids = 3;
  // True means the field is searchable (indexed) in at least some indices.
  // False means the field is not searchable in any indices.
  bool searchable = 4;
  // True means the field is aggregatable (fast) in at least some indices.
  // False means the field is not aggregatable in any indices.
  bool aggregatable = 5;
  // The index ids the field exists, but is not searchable.
  repeated string non_searchable_index_ids = 6;
  // The index ids the field exists, but is not aggregatable
  repeated string non_aggregatable_index_ids = 7;
}

enum ListFieldType {
    STR = 0;
    U64 = 1;
    I64 = 2;
    F64 = 3;
    BOOL = 4;
    DATE = 5;
    FACET = 6;
    BYTES = 7;
    IP_ADDR = 8;
    JSON = 9;
}
message ListFields {
  repeated ListFieldsEntryResponse fields = 1;
}
// -- Search -------------------

message SearchRequest {
  // Index ID patterns
  repeated string index_id_patterns = 1;

  // deprecated `query``
  reserved 2;

  // Json object representing Quickwit's QueryAst.
  string query_ast = 13;

  // deprecated `search_fields``
  reserved 3;

  // Time filter, expressed in seconds since epoch.
  // That filter is to be interpreted as the semi-open interval:
  // [start_timestamp, end_timestamp).
  optional int64 start_timestamp = 4;
  optional int64 end_timestamp = 5;

  // Maximum number of hits to return.
  uint64 max_hits = 6;

  // First hit to return. Together with max_hits, this parameter
  // can be used for pagination.
  //
  // E.g.
  // The results with rank [start_offset..start_offset + max_hits) are returned.
  uint64 start_offset = 7;

  // deprecated tag field
  reserved 8;

  // deprecated `sort_order``
  reserved 9;

  // deprecated `sort_by_field``
  reserved 10;

  // json serialized aggregation_request
  optional string aggregation_request = 11;

  // Fields to extract snippet on
  repeated string snippet_fields = 12;

  // Optional sort by one or more fields (limited to 2 at the moment).
  repeated SortField sort_fields = 14;

  // If set, the search response will include a search id
  // that will make it possible to paginate through the results
  // in a consistent manner.
  optional uint32 scroll_ttl_secs = 15;

  // Document with sort tuple smaller or equal to this are discarded to
  // enable pagination.
  // If split_id is empty, no comparison with _shard_doc should be done
  optional PartialHit search_after = 16;

  CountHits count_hits = 17;
}

enum CountHits {
  // Count all hits, querying all splits.
  COUNT_ALL = 0;
  // Give an underestimate of the number of hits, possibly skipping entire
  // splits if they are otherwise not needed to fulfull a query.
  UNDERESTIMATE = 1;
}

message SortField {
  string field_name = 1;
  SortOrder sort_order = 2;
  // Optional sort value format for datetime field only.
  // If none, the default output format for datetime field is
  // unix_timestamp_nanos.
  optional SortDatetimeFormat sort_datetime_format = 3;
}

enum SortOrder {
  // Ascending order.
  ASC = 0;
  // Descending order.
  DESC = 1; //< This will be the default value;
}

// Sort value format for datetime field.
// We keep an enum with only one format
// for future extension.
enum SortDatetimeFormat {
  UNIX_TIMESTAMP_MILLIS = 0;
  UNIX_TIMESTAMP_NANOS = 1;
}

message SearchResponse {
  // Number of hits matching the query.
  uint64 num_hits = 1;
  // Matched hits
  repeated Hit hits = 2;
  // Elapsed time to perform the request. This time is measured
  // server-side and expressed in microseconds.
  uint64 elapsed_time_micros = 3;

  // The searcherrors that occurred formatted as string.
  repeated string errors = 4;

  // Serialized aggregation response
  optional string aggregation = 5;

  // Scroll Id (only set if scroll_secs was set in the request)
  optional string scroll_id = 6;

  // Returns the list of splits for which search failed.
  // For the moment, the cause is unknown.
  //
  // It is up to the caller to decide whether to interpret
  // this as an overall failure or to present the partial results
  // to the end user.
  repeated SplitSearchError failed_splits = 7;

  // Total number of successful splits searched.
  uint64 num_successful_splits = 8;
}

message SearchPlanResponse {
  string result = 1;
}

message SplitSearchError {
  // The searcherror that occurred formatted as string.
  string error = 1;

  // Split id that failed.
  string split_id = 2;

  // Flag to indicate if the error can be considered a retryable error
  bool retryable_error = 3;
}

/// A LeafSearchRequest can span multiple indices.
///
message LeafSearchRequest {
  // Search request. This is a perfect copy of the original search request
  // that was sent to root apart from the start_offset, max_hits params and index_id_patterns.
  // index_id_patterns contains the actual index ids queried on that leaf.
  SearchRequest search_request = 1;

  // List of leaf requests, one per index.
  repeated LeafRequestRef leaf_requests = 7;

  // List of unique doc_mappers serialized as json.
  repeated string doc_mappers = 8;

  // List of index uris
  // Index URI. The index URI defines the location of the storage that contains the
  // split files.
  repeated string index_uris = 9;
}

/// LeafRequestRef references data in LeafSearchRequest to deduplicate data.
message LeafRequestRef {
  // The ordinal of the doc_mapper in `LeafSearchRequest.doc_mappers`
  uint32 doc_mapper_ord = 1;

  // The ordinal of the index uri in LeafSearchRequest.index_uris
  uint32 index_uri_ord = 2;

  // Index split ids to apply the query on.
  // This ids are resolved from the index_uri defined in the search_request.
  repeated SplitIdAndFooterOffsets split_offsets = 3;
}

message SplitIdAndFooterOffsets {
  // Index split id to apply the query on.
  // This id is resolved from the index_uri defined in the search_request.
  string split_id = 1;
  // The offset of the start of footer in the split bundle. The footer contains the file bundle metadata and the hotcache.
  uint64 split_footer_start = 2;
  // The offset of the end of the footer in split bundle. The footer contains the file bundle metadata and the hotcache.
  uint64 split_footer_end = 3;
  // The lowest timestamp appearing in the split, in seconds since epoch
  optional int64 timestamp_start = 4;
  // The highest timestamp appearing in the split, in seconds since epoch
  optional int64 timestamp_end = 5;
  // The number of docs in the split
  uint64 num_docs = 6;
}

// Hits returned by a FetchDocRequest.
//
// The json that is joined is the raw tantivy json doc.
// It is very different from a quickwit json doc.
//
// For instance:
// - it may contain a _source and a _dynamic field.
// - since tantivy has no notion of cardinality,
//   all fields are arrays.
// - since tantivy has no notion of object, the object is
//   flattened by concatenating the path to the root.
//
// See  `quickwit_search::convert_leaf_hit`
message LeafHit {
  // The actual content of the hit/
  string leaf_json = 1;
  // The partial hit (ie: the sorting field + the document address)
  PartialHit partial_hit = 2;
  // A snippet of the matching content
  optional string leaf_snippet_json = 3;
}

message Hit {
  // The actual content of the hit
  string json = 1;
  // The partial hit (ie: the sorting field + the document address)
  PartialHit partial_hit = 2;
  // A snippet of the matching content
  optional string snippet = 3;
  // The index id of the hit
  string index_id = 4;
}


// A partial hit, is a hit for which we have not fetch the content yet.
// Instead, it holds a document_uri which is enough information to
// go and fetch the actual document data, by performing a `get_doc(...)`
// request.
message PartialHit {
  // Value of the sorting key for the given document.
  //
  // Quickwit only computes top-K of this sorting field.
  // If the user requested for a bottom-K of a given fast field, then quickwit simply
  // emits an decreasing mapping of this fast field.
  //
  // In case of a tie, quickwit uses the increasing order of
  // - the split_id,
  // - the segment_ord,
  // - the doc id.

  // Deprecated
  reserved 1;
  // Room for eventual future sorted key types.
  reserved 12 to 20;
  SortByValue sort_value = 10;
  SortByValue sort_value2 = 11;

  string split_id = 2;

  // (segment_ord, doc) form a tantivy DocAddress, which is sufficient to identify a document
  // within a split
  uint32 segment_ord = 3;

  // The DocId identifies a unique document at the scale of a tantivy segment.
  uint32 doc_id = 4;
}

message SortByValue {
  oneof sort_value {
  uint64 u64 = 1;
  int64 i64 = 2;
  double f64 = 3;
  bool boolean = 4;
  }
  // Room for eventual future sorted key types.
  reserved 5 to 20;
}

message LeafSearchResponse {
  // Total number of documents matched by the query.
  uint64 num_hits = 1;

  // List of the best top-K candidates for the given leaf query.
  repeated PartialHit partial_hits = 2;

  // The list of splits that failed. LeafSearchResponse can be an aggregation of results, so there may be multiple.
  repeated SplitSearchError failed_splits = 3;

  // Total number of attempt to search into splits.
  // We do have:
  // `num_splits_requested == num_successful_splits + num_failed_splits.len()`
  // But we do not necessarily have:
  // `num_splits_requested = num_attempted_splits because of retries.`
  uint64 num_attempted_splits = 4;

  // Total number of successful splits searched.
  uint64 num_successful_splits = 7;

  // Deprecated json serialized intermediate aggregation_result.
  reserved 5;

  // postcard serialized intermediate aggregation_result.
  optional bytes intermediate_aggregation_result = 6;
}

message SnippetRequest {
  repeated string snippet_fields = 1;
  string query_ast_resolved = 2;
}

message FetchDocsRequest {
  // Request fetching the content of a given list of partial_hits.
  repeated PartialHit partial_hits = 1;

  // Split footer offsets. They are required for fetch docs to
  // fetch the document content in two reads, when the footer is not
  // cached.
  repeated SplitIdAndFooterOffsets split_offsets = 3;

  // Index URI. The index URI defines the location of the storage that contains the
  // split files.
  string index_uri = 4;

  optional SnippetRequest snippet_request = 7;

  // `DocMapper` as json serialized trait.
  string doc_mapper = 6;

  reserved 5;
}

message FetchDocsResponse {
  // List of complete hits.
  repeated LeafHit hits = 1;
}

message ListTermsRequest {
  // Index ID patterns
  repeated string index_id_patterns = 1;

  // Field to search on
  string field = 3;

  // Time filter
  optional int64 start_timestamp = 4;
  optional int64 end_timestamp = 5;

  // Maximum number of hits to return.
  optional uint64 max_hits = 6;

  // start_key is included, end_key is excluded
  optional bytes start_key = 7;
  optional bytes end_key = 8;
}

message ListTermsResponse {
  // Number of hits matching the query.
  uint64 num_hits = 1;
  // Matched hits
  repeated bytes terms = 2;
  // Elapsed time to perform the request. This time is measured
  // server-side and expressed in microseconds.
  uint64 elapsed_time_micros = 3;

  // The searcherrors that occurred formatted as string.
  repeated string errors = 4;
}

message LeafListTermsRequest {
  // Search request. This is a perfect copy of the original list request,
  ListTermsRequest list_terms_request = 1;

  // Index split ids to apply the query on.
  // This ids are resolved from the index_uri defined in the search_request.
  repeated SplitIdAndFooterOffsets split_offsets = 2;

  // Index URI. The index URI defines the location of the storage that contains the
  // split files.
  string index_uri = 3;
}

message LeafListTermsResponse {
  // Total number of documents matched by the query.
  uint64 num_hits = 1;

  // List of the first K terms the given leaf query.
  repeated bytes terms = 2;

  // The list of splits that failed. LeafSearchResponse can be an aggregation of results, so there may be multiple.
  repeated SplitSearchError failed_splits = 3;

  // Total number of single split search attempted.
  uint64 num_attempted_splits = 4;
}

// -- Stream -------------------

enum OutputFormat {
  // Comma Separated Values format (https://datatracker.ietf.org/doc/html/rfc4180).
  // The delimiter is `,`.
  CSV = 0; //< This will be the default value
  // Format data by row in ClickHouse binary format.
  // https://clickhouse.tech/docs/en/interfaces/formats/#rowbinary
  CLICK_HOUSE_ROW_BINARY = 1;
}

message SearchStreamRequest {
  // Index ID
  string index_id = 1;

  // deprecated `query`.
  reserved 2;

  // Quickwit Query AST encoded in Json
  string query_ast = 11;

  // deprecated `search_fields`
  reserved 3;

  // The time filter is interpreted as a semi-open interval. [start, end)
  optional int64 start_timestamp = 4;
  optional int64 end_timestamp = 5;

  // Name of the fast field to extract
  string fast_field = 6;

  // The output format
  OutputFormat output_format = 7;

  reserved 8; // deprecated field: tags

  // The field by which we want to partition
  optional string partition_by_field = 9;

  // Fields to extract snippet on.
  repeated string snippet_fields = 10;
}

message LeafSearchStreamRequest {
  // Stream request. This is a perfect copy of the original stream request,
  // that was sent to root.
  SearchStreamRequest request = 1;

  // Index split ids to apply the query on.
  // This ids are resolved from the index_uri defined in the stream request.
  repeated SplitIdAndFooterOffsets split_offsets = 2;

  // `DocMapper` as json serialized trait.
  string doc_mapper = 5;

  // Index URI. The index URI defines the location of the storage that contains the
  // split files.
  string index_uri = 6;

}


message LeafSearchStreamResponse {
  // Row of data serialized in bytes.
  bytes data = 1;

  // Split id.
  string split_id = 2;
}
