Client SDK, CLI and Python for secondary indexes (refs #180)

api/kv_operation.py

@@ -24,6 +24,8 @@
 parent_dir = os.path.dirname(current_dir)
 new_path_dir = os.path.join(parent_dir, "bazel-out", "k8-fastbuild", "bin", "api")
 sys.path.insert(0, new_path_dir)
+# bazel-bin points at the build output for whatever platform was built.
+sys.path.insert(0, os.path.join(parent_dir, "bazel-bin", "api"))
 import pybind_kv
 
 
@@ -44,3 +46,71 @@ def get_value(key: str or int or float, config_path: str = current_dir + "/ip_ad
     :return: A string of the key's corresponding value.
     """
     return pybind_kv.get(str(key), os.path.abspath(config_path))
+
+
+def create_index_entry(index_name: str, attributes, primary_key: str,
+                       config_path: str = current_dir + "/ip_address.config") -> int:
+    """
+    Add a secondary index entry, e.g. create_index_entry("by_city", "Davis", "user:1").
+
+    :param index_name: The index the entry belongs to, chosen by you.
+    :param attributes: One attribute value, or a list of them for a multi-attribute index.
+                       Mixing the two forms in one call is not supported.
+    :param primary_key: The key of the record being indexed. It must already exist.
+    :param config_path: Path to the client config.
+    :return: 0 on success, -3 if the servers rejected it, -1 or -2 if it could not be sent.
+    """
+    return pybind_kv.create_index_entry(index_name, attributes, str(primary_key),
+                                        os.path.abspath(config_path))
+
+
+def delete_index_entry(index_name: str, attributes, primary_key: str,
+                       config_path: str = current_dir + "/ip_address.config") -> int:
+    """
+    Remove a secondary index entry. The record itself is not deleted.
+
+    :return: 0 on success, -3 if the servers rejected it (no such entry, empty index
+             name or primary key, or a zero byte in any part), -1 or -2 if it could not be sent.
+    """
+    return pybind_kv.delete_index_entry(index_name, attributes, str(primary_key),
+                                        os.path.abspath(config_path))
+
+
+def update_index_entry(index_name: str, old_attributes, new_attributes, primary_key: str,
+                       config_path: str = current_dir + "/ip_address.config") -> int:
+    """
+    Move an entry to new attributes, e.g. when a record's city changes. The old entry is
+    removed and the new one added in a single write.
+
+    :return: 0 on success, -3 if the servers rejected it (the old entry does not exist,
+             or the input was invalid), -1 or -2 if it could not be sent.
+    """
+    return pybind_kv.update_index_entry(index_name, old_attributes, new_attributes,
+                                        str(primary_key), os.path.abspath(config_path))
+
+
+def query_by_index(index_name: str, attributes,
+                   config_path: str = current_dir + "/ip_address.config") -> list:
+    """
+    Primary keys whose entry starts with the given attributes, each once and sorted.
+
+    Attributes match from the left: in an index built as (owner, format), you can query
+    by owner, or by owner and format, but not by format alone. An empty list returns
+    every entry in the index.
+
+    :return: A list of primary keys, empty if nothing matched or the request failed.
+    """
+    return pybind_kv.query_by_index(index_name, attributes, os.path.abspath(config_path))
+
+
+def query_records_by_index(index_name: str, attributes,
+                           config_path: str = current_dir + "/ip_address.config") -> list:
+    """
+    Like query_by_index, but returns (key, value) pairs. The records are read on the
+    server and come back in the same reply, instead of one request per match.
+
+    :return: A list of (primary_key, value) tuples, empty if nothing matched or the
+             request failed.
+    """
+    return pybind_kv.query_records_by_index(index_name, attributes,
+                                            os.path.abspath(config_path))

api/pybind_kv_service.cpp

@@ -20,11 +20,14 @@
 #include <fcntl.h>
 #include <getopt.h>
 #include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <fstream>
+#include <string>
+#include <vector>
 
 #include "common/proto/signature_info.pb.h"
 #include "interface/kv/kv_client.h"
@@ -60,7 +63,126 @@ bool set(std::string key, std::string value, std::string config_path) {
   }
 }
 
+namespace {
+
+KVClient MakeClient(const std::string& config_path) {
+  ResDBConfig config = GenerateResDBConfig(config_path);
+  config.SetClientTimeoutMs(100000);
+  return KVClient(config);
+}
+
+}  // namespace
+
+// Index commands. Return 0 on success, -3 if the servers rejected the
+// request, or a send error (-1, -2). Attributes may be a list or, for a
+// single-attribute index, a plain string.
+int create_index_entry(std::string index_name,
+                       std::vector<std::string> attributes,
+                       std::string primary_key, std::string config_path) {
+  return MakeClient(config_path)
+      .CreateIndexEntry(index_name, attributes, primary_key);
+}
+
+int create_index_entry_one(std::string index_name, std::string attribute,
+                           std::string primary_key, std::string config_path) {
+  return create_index_entry(index_name, {attribute}, primary_key, config_path);
+}
+
+int delete_index_entry(std::string index_name,
+                       std::vector<std::string> attributes,
+                       std::string primary_key, std::string config_path) {
+  return MakeClient(config_path)
+      .DeleteIndexEntry(index_name, attributes, primary_key);
+}
+
+int delete_index_entry_one(std::string index_name, std::string attribute,
+                           std::string primary_key, std::string config_path) {
+  return delete_index_entry(index_name, {attribute}, primary_key, config_path);
+}
+
+int update_index_entry(std::string index_name,
+                       std::vector<std::string> old_attributes,
+                       std::vector<std::string> new_attributes,
+                       std::string primary_key, std::string config_path) {
+  return MakeClient(config_path)
+      .UpdateIndexEntry(index_name, old_attributes, new_attributes,
+                        primary_key);
+}
+
+int update_index_entry_one(std::string index_name, std::string old_attribute,
+                           std::string new_attribute, std::string primary_key,
+                           std::string config_path) {
+  return update_index_entry(index_name, {old_attribute}, {new_attribute},
+                            primary_key, config_path);
+}
+
+// Primary keys whose entry starts with `attributes`, each once and sorted.
+// An empty list means the whole index.
+std::vector<std::string> query_by_index(std::string index_name,
+                                        std::vector<std::string> attributes,
+                                        std::string config_path) {
+  std::vector<std::string> keys;
+  auto items = MakeClient(config_path).QueryByIndex(index_name, attributes);
+  if (items == nullptr) {
+    return keys;
+  }
+  for (const auto& item : items->item()) {
+    keys.push_back(item.key());
+  }
+  return keys;
+}
+
+std::vector<std::string> query_by_index_one(std::string index_name,
+                                            std::string attribute,
+                                            std::string config_path) {
+  return query_by_index(index_name, {attribute}, config_path);
+}
+
+// Same query, but each record's value comes back in the same reply instead of
+// one request per match.
+std::vector<std::pair<std::string, std::string>> query_records_by_index(
+    std::string index_name, std::vector<std::string> attributes,
+    std::string config_path) {
+  std::vector<std::pair<std::string, std::string>> records;
+  auto items =
+      MakeClient(config_path).QueryByIndex(index_name, attributes, true);
+  if (items == nullptr) {
+    return records;
+  }
+  for (const auto& item : items->item()) {
+    records.emplace_back(item.key(), item.value_info().value());
+  }
+  return records;
+}
+
+std::vector<std::pair<std::string, std::string>> query_records_by_index_one(
+    std::string index_name, std::string attribute, std::string config_path) {
+  return query_records_by_index(index_name, {attribute}, config_path);
+}
+
 PYBIND11_MODULE(pybind_kv, m) {
   m.def("get", &get, "A function that gets a value from the key-value store");
   m.def("set", &set, "A function that sets a value in the key-value store");
+  // Each command has a single-string and a list overload: pybind never turns a
+  // str into a list of strings, so "Davis" needs its own signature.
+  m.def("create_index_entry", &create_index_entry_one,
+        "Add an index entry with one attribute");
+  m.def("create_index_entry", &create_index_entry,
+        "Add an index entry with a list of attributes");
+  m.def("delete_index_entry", &delete_index_entry_one,
+        "Remove an index entry with one attribute");
+  m.def("delete_index_entry", &delete_index_entry,
+        "Remove an index entry with a list of attributes");
+  m.def("update_index_entry", &update_index_entry_one,
+        "Move an index entry to a new attribute");
+  m.def("update_index_entry", &update_index_entry,
+        "Move an index entry to new attributes");
+  m.def("query_by_index", &query_by_index_one,
+        "Primary keys matching one attribute");
+  m.def("query_by_index", &query_by_index,
+        "Primary keys matching a list of attributes");
+  m.def("query_records_by_index", &query_records_by_index_one,
+        "(key, value) pairs matching one attribute");
+  m.def("query_records_by_index", &query_records_by_index,
+        "(key, value) pairs matching a list of attributes");
 }

benchmark/e2e/BUILD

@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "composite_key_e2e_benchmark",
+    srcs = ["composite_key_e2e_benchmark.cpp"],
+    deps = [
+        "//interface/kv:kv_client",
+        "//platform/config:resdb_config_utils",
+    ],
+)