From 3a2abf93050279500d6a3b3b1517209f627c5ea8 Mon Sep 17 00:00:00 2001 From: Rebekah Davis Date: Mon, 18 Nov 2024 18:15:00 -0500 Subject: [PATCH 1/2] Add Config::login API. --- tiledb/sm/config/config.cc | 103 ++++++++++++++++++--------- tiledb/sm/config/config.h | 5 +- tiledb/sm/config/test/unit_config.cc | 15 +++- 3 files changed, 88 insertions(+), 35 deletions(-) diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index d879122e6c5..e6c3d49f28e 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -30,16 +30,20 @@ * This file implements class Config. */ +#include #include #include #include #include "config.h" +#include "external/include/nlohmann/json.hpp" #include "tiledb/common/logger.h" #include "tiledb/sm/enums/serialization_type.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/parse_argument.h" +using json = nlohmann::json; + using namespace tiledb::common; namespace { @@ -60,14 +64,12 @@ bool ignore_default_via_env(const std::string& param) { namespace tiledb::sm { -/** Return a Config error class Status with a given message **/ -inline Status Status_ConfigError(const std::string& msg) { - return {"[TileDB::Config] Error", msg}; -} - -void throw_config_exception(const std::string& msg) { - throw StatusException(Status_ConfigError(msg)); -} +class ConfigException : public StatusException { + public: + explicit ConfigException(const std::string& msg) + : StatusException("Config", msg) { + } +}; /* ****************************** */ /* CONFIG DEFAULTS */ @@ -561,17 +563,56 @@ Config::~Config() = default; /* API */ /* ****************************** */ +void Config::login() { + // Retrieve $HOME path + std::string home_dir; +#ifdef _WIN32 + home_dir = + std::string(std::getenv("HOMEDRIVE")) + std::string(getenv("HOMEPATH")); +#else + home_dir = std::string(std::getenv("HOME")); +#endif + + // For library versions 22 and older, simply parse the local .json file + if (constants::format_version <= 22) { + // Find and parse the cloud.json file + std::string file = home_dir + "/.tiledb/cloud.json"; + if (!std::filesystem::exists(file)) { + throw ConfigException("Cannot login; cloud.json file does not exist."); + } + json data = json::parse(std::ifstream(file)); + + // Set the config values that have been saved to the file + if (data.contains("api_key") && + data["api_key"].contains("X-TILEDB-REST-API-KEY")) { + throw_if_not_ok( + set("rest.token", data["api_key"]["X-TILEDB-REST-API-KEY"])); + } + if (data.contains("host")) { + throw_if_not_ok(set("rest.server_address", data["host"])); + } + if (data.contains("password")) { + throw_if_not_ok(set("rest.password", data["password"])); + } + if (data.contains("username")) { + throw_if_not_ok(set("rest.username", data["username"])); + } + if (data.contains("verify_ssl")) { + throw_if_not_ok( + set("vfs.s3.verify_ssl", data["verify_ssl"] ? "true" : "false")); + } + } +} + Status Config::load_from_file(const std::string& filename) { // Do nothing if filename is empty - if (filename.empty()) - return LOG_STATUS( - Status_ConfigError("Cannot load from file; Invalid filename")); + if (filename.empty()) { + throw ConfigException("Cannot load from file; Invalid filename"); + } std::ifstream ifs(filename); if (!ifs.is_open()) { - std::stringstream msg; - msg << "Failed to open config file '" << filename << "'"; - return LOG_STATUS(Status_ConfigError(msg.str())); + throw ConfigException("Failed to open config file '" + filename + "'"); } size_t linenum = 0; @@ -592,7 +633,7 @@ Status Config::load_from_file(const std::string& filename) { std::stringstream msg; msg << "Failed to parse config file '" << filename << "'; "; msg << "Missing parameter value (line: " << linenum << ")"; - return LOG_STATUS(Status_ConfigError(msg.str())); + throw ConfigException(msg.str()); } // Parse extra @@ -601,7 +642,7 @@ Status Config::load_from_file(const std::string& filename) { std::stringstream msg; msg << "Failed to parse config file '" << filename << "'; "; msg << "Invalid line format (line: " << linenum << ")"; - return LOG_STATUS(Status_ConfigError(msg.str())); + throw ConfigException(msg.str()); } // Set param-value pair @@ -614,15 +655,14 @@ Status Config::load_from_file(const std::string& filename) { Status Config::save_to_file(const std::string& filename) { // Do nothing if filename is empty - if (filename.empty()) - return LOG_STATUS( - Status_ConfigError("Cannot save to file; Invalid filename")); + if (filename.empty()) { + throw ConfigException("Cannot save to file; Invalid filename"); + } std::ofstream ofs(filename); if (!ofs.is_open()) { - std::stringstream msg; - msg << "Failed to open config file '" << filename << "' for writing"; - return LOG_STATUS(Status_ConfigError(msg.str())); + throw ConfigException( + "Failed to open config file '" + filename + "' for writing"); } for (auto& pv : param_values_) { if (unserialized_params_.count(pv.first) != 0) @@ -664,7 +704,7 @@ Status Config::get(const std::string& param, T* value, bool* found) const { // Parameter found, retrieve value auto status = utils::parse::convert(val, value); if (!status.ok()) { - return Status_ConfigError( + throw ConfigException( std::string("Failed to parse config value '") + std::string(val) + std::string("' for key '") + param + "' due to: " + status.to_string()); } @@ -751,8 +791,7 @@ Status Config::sanity_check( RETURN_NOT_OK(utils::parse::convert(value, &v32)); } else if (param == "config.logging_format") { if (value != "DEFAULT" && value != "JSON") - return LOG_STATUS( - Status_ConfigError("Invalid logging format parameter value")); + throw ConfigException("Invalid logging format parameter value"); } else if (param == "sm.allow_separate_attribute_writes") { RETURN_NOT_OK(utils::parse::convert(value, &v)); } else if (param == "sm.allow_updates_experimental") { @@ -799,8 +838,7 @@ Status Config::sanity_check( RETURN_NOT_OK(utils::parse::convert(value, &v)); } else if (param == "sm.var_offsets.mode") { if (value != "bytes" && value != "elements") - return LOG_STATUS( - Status_ConfigError("Invalid offsets format parameter value")); + throw ConfigException("Invalid offsets format parameter value"); } else if (param == "sm.fragment_info.preload_mbrs") { RETURN_NOT_OK(utils::parse::convert(value, &v)); } else if (param == "ssl.verify") { @@ -825,8 +863,7 @@ Status Config::sanity_check( RETURN_NOT_OK(utils::parse::convert(value, &v32)); } else if (param == "vfs.s3.scheme") { if (value != "http" && value != "https") - return LOG_STATUS( - Status_ConfigError("Invalid S3 scheme parameter value")); + throw ConfigException("Invalid S3 scheme parameter value"); } else if (param == "vfs.s3.use_virtual_addressing") { RETURN_NOT_OK(utils::parse::convert(value, &v)); } else if (param == "vfs.s3.skit_init") { @@ -866,7 +903,7 @@ Status Config::sanity_check( (value == "bucket_owner_full_control"))))) { std::stringstream msg; msg << "value " << param << " invalid canned acl for " << param; - return Status_Error(msg.str()); + throw ConfigException(msg.str()); } } @@ -970,7 +1007,7 @@ optional Config::get_internal(const std::string& key) const { if (status.ok()) { return {converted_value}; } - throw_config_exception( + throw ConfigException( "Failed to parse config value '" + std::string(value.value()) + "' for key '" + key + "'. Reason: " + status.to_string()); } @@ -988,7 +1025,7 @@ optional Config::get_internal_string( } if constexpr (must_find_) { - throw_config_exception("Failed to get config value for key: " + key); + throw ConfigException("Failed to get config value for key: " + key); } return {nullopt}; } diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index 1f75e6f97a0..2d98d012dd5 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -667,6 +667,9 @@ class Config { /* API */ /* ********************************* */ + /** Log in to TileDB Cloud, saving profile info to the config. */ + void login(); + /** Loads the config parameters from a configuration (local) file. */ Status load_from_file(const std::string& filename); diff --git a/tiledb/sm/config/test/unit_config.cc b/tiledb/sm/config/test/unit_config.cc index 5f161cd1fa8..b5800329633 100644 --- a/tiledb/sm/config/test/unit_config.cc +++ b/tiledb/sm/config/test/unit_config.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -111,3 +111,16 @@ TEST_CASE("Config::get - found and matched", "[config]") { CHECK(c.set(key, expected_value).ok()); TestConfig::check_expected(expected_value, c, key); } + +TEST_CASE("Config::login", "[config][login]") { + Config c{}; + + // Upon initial Config construction, no rest token is set + auto initial_token = c.get("rest.token"); + CHECK(!initial_token.has_value()); + + // After login, the rest token is set from the cloud.json file + c.login(); + auto login_token = c.get("rest.token"); + CHECK(login_token.has_value()); +} From 640698088aef1b3426de6552e540093c69bf9fa5 Mon Sep 17 00:00:00 2001 From: Rebekah Davis Date: Thu, 21 Nov 2024 14:00:31 -0500 Subject: [PATCH 2/2] Add Config::set_internal API. --- test/src/unit-capi-config.cc | 22 +++++++++++++++------- test/src/unit-curl.cc | 6 +++--- tiledb/sm/config/config.cc | 26 +++++++++++++++----------- tiledb/sm/config/config.h | 10 ++++++++++ tiledb/sm/config/test/unit_config.cc | 15 +-------------- 5 files changed, 44 insertions(+), 35 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 36e52835416..b9438d9419d 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB Inc. + * @copyright Copyright (c) 2017-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,6 +40,7 @@ #include #include #include +#include void remove_file(const std::string& filename) { // Remove file @@ -98,9 +99,7 @@ void check_load_incorrect_file_cannot_open() { rc = tiledb_config_load_from_file(config, "non_existent_file", &error); CHECK(rc == TILEDB_ERR); CHECK(error != nullptr); - check_error( - error, - "[TileDB::Config] Error: Failed to open config file 'non_existent_file'"); + check_error(error, "Config: Failed to open config file 'non_existent_file'"); tiledb_error_free(&error); tiledb_config_free(&config); CHECK(config == nullptr); @@ -127,7 +126,7 @@ void check_load_incorrect_file_missing_value() { CHECK(error != nullptr); check_error( error, - "[TileDB::Config] Error: Failed to parse config file 'test_config.txt'; " + "Config: Failed to parse config file 'test_config.txt'; " "Missing parameter value (line: 1)"); tiledb_error_free(&error); CHECK(error == nullptr); @@ -157,7 +156,7 @@ void check_load_incorrect_file_extra_word() { CHECK(error != nullptr); check_error( error, - "[TileDB::Config] Error: Failed to parse config file 'test_config.txt'; " + "Config: Failed to parse config file 'test_config.txt'; " "Invalid line format (line: 3)"); tiledb_error_free(&error); tiledb_config_free(&config); @@ -906,6 +905,12 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { s3_param_values["config_source"] = "auto"; s3_param_values["install_sigpipe_handler"] = "true"; + // A list of "sensitive" parameters, whose key-values should not be exposed. + std::unordered_set sensitive_param_values; + sensitive_param_values.emplace("rest.token"); + sensitive_param_values.emplace("rest.username"); + sensitive_param_values.emplace("rest.password"); + // Create an iterator and iterate over all parameters tiledb_config_iter_t* config_iter = nullptr; rc = tiledb_config_iter_alloc(config, nullptr, &config_iter, &error); @@ -924,7 +929,10 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { CHECK(error == nullptr); CHECK(param != nullptr); CHECK(value != nullptr); - all_iter_map[std::string(param)] = std::string(value); + // Skip checks for sensitive params to avoid exposing their values. + if (!sensitive_param_values.contains(std::string(param))) { + all_iter_map[std::string(param)] = std::string(value); + } rc = tiledb_config_iter_next(config_iter, &error); CHECK(rc == TILEDB_OK); CHECK(error == nullptr); diff --git a/test/src/unit-curl.cc b/test/src/unit-curl.cc index e24deb32ab8..bd551969731 100644 --- a/test/src/unit-curl.cc +++ b/test/src/unit-curl.cc @@ -129,13 +129,13 @@ TEST_CASE( "RestClient: Remove trailing slash from rest_server_", "[rest-client]") { std::string rest_server = GENERATE("http://localhost:8080/", "http://localhost:8080//"); + SECTION("rest.server_address set in environment") { + setenv_local("TILEDB_REST_SERVER_ADDRESS", rest_server.c_str()); + } tiledb::sm::Config cfg; SECTION("rest.server_address set in Config") { cfg.set("rest.server_address", rest_server).ok(); } - SECTION("rest.server_address set in environment") { - setenv_local("TILEDB_REST_SERVER_ADDRESS", rest_server.c_str()); - } SECTION("rest.server_address set by loaded config file") { std::string cfg_file = "tiledb_config.txt"; std::ofstream file(cfg_file); diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index e6c3d49f28e..14ebc8aeca0 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -52,7 +52,6 @@ bool ignore_default_via_env(const std::string& param) { // We should not use the default value for `vfs.s3.region` if the user // has set either AWS_REGION or AWS_DEFAULT_REGION in their environment. // We defer to the SDK to interpret these values. - if ((std::getenv("AWS_REGION") != nullptr) || (std::getenv("AWS_DEFAULT_REGION") != nullptr)) { return true; @@ -555,6 +554,7 @@ const std::set Config::unserialized_params_ = { Config::Config() { // Set config values param_values_ = default_config_values; + login(); } Config::~Config() = default; @@ -573,33 +573,32 @@ void Config::login() { home_dir = std::string(std::getenv("HOME")); #endif - // For library versions 22 and older, simply parse the local .json file - if (constants::format_version <= 22) { + // For library versions 2.27.0 and older, simply parse the local .json file + auto version = constants::library_version; + if (version[0] <= 2 && version[1] <= 27) { // Find and parse the cloud.json file std::string file = home_dir + "/.tiledb/cloud.json"; if (!std::filesystem::exists(file)) { - throw ConfigException("Cannot login; cloud.json file does not exist."); + return; } json data = json::parse(std::ifstream(file)); // Set the config values that have been saved to the file if (data.contains("api_key") && data["api_key"].contains("X-TILEDB-REST-API-KEY")) { - throw_if_not_ok( - set("rest.token", data["api_key"]["X-TILEDB-REST-API-KEY"])); + set_internal("rest.token", data["api_key"]["X-TILEDB-REST-API-KEY"]); } if (data.contains("host")) { - throw_if_not_ok(set("rest.server_address", data["host"])); + set_internal("rest.server_address", data["host"]); } if (data.contains("password")) { - throw_if_not_ok(set("rest.password", data["password"])); + set_internal("rest.password", data["password"]); } if (data.contains("username")) { - throw_if_not_ok(set("rest.username", data["username"])); + set_internal("rest.username", data["username"]); } if (data.contains("verify_ssl")) { - throw_if_not_ok( - set("vfs.s3.verify_ssl", data["verify_ssl"] ? "true" : "false")); + set_internal("vfs.s3.verify_ssl", data["verify_ssl"] ? "true" : "false"); } } } @@ -1030,6 +1029,11 @@ optional Config::get_internal_string( return {nullopt}; } +void Config::set_internal(const std::string& param, const std::string& value) { + throw_if_not_ok(sanity_check(param, value)); + param_values_[param] = value; +} + /* * Explicit instantiations */ diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index 2d98d012dd5..02a05a6e06d 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -827,6 +827,16 @@ class Config { /** Returns the param -> value map. */ const std::map& param_values() const; + + /** + * Internally sets the given config parameter. + * + * @note For internal use only; This API does not update the user-set params. + * + * @param param The config parameter to set. + * @param value The value of the parameter. + */ + void set_internal(const std::string& param, const std::string& value); }; /** diff --git a/tiledb/sm/config/test/unit_config.cc b/tiledb/sm/config/test/unit_config.cc index b5800329633..5f161cd1fa8 100644 --- a/tiledb/sm/config/test/unit_config.cc +++ b/tiledb/sm/config/test/unit_config.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2024 TileDB, Inc. + * @copyright Copyright (c) 2022 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -111,16 +111,3 @@ TEST_CASE("Config::get - found and matched", "[config]") { CHECK(c.set(key, expected_value).ok()); TestConfig::check_expected(expected_value, c, key); } - -TEST_CASE("Config::login", "[config][login]") { - Config c{}; - - // Upon initial Config construction, no rest token is set - auto initial_token = c.get("rest.token"); - CHECK(!initial_token.has_value()); - - // After login, the rest token is set from the cloud.json file - c.login(); - auto login_token = c.get("rest.token"); - CHECK(login_token.has_value()); -}