use chrono::Utc;
use clap::{CommandFactory, Parser, ValueEnum};
use clap_complete::CompleteEnv;
use feldera_rest_api::types::*;
use feldera_rest_api::*;
use feldera_types::checkpoint::CheckpointResponse;
use feldera_types::config::{FtModel, RuntimeConfig, StorageOptions};
use feldera_types::duration::ConfigDuration;
use feldera_types::error::ErrorResponse;
use feldera_types::size::ConfigSize;
use feldera_types::transport::clock::ClockAdvanceRequest;
use futures_util::StreamExt;
use json_to_table::json_to_table;
use log::{debug, error, info, trace, warn};
use progenitor_client::ClientInfo;
use reqwest::StatusCode;
use reqwest::header::{HeaderMap, HeaderValue, InvalidHeaderValue};
use serde_json::json;
use std::convert::Infallible;
use std::fmt::Display;
use std::fmt::Write as _;
use std::fs::File;
use std::io::{ErrorKind, Read, Write, stdout};
use std::path::PathBuf;
use std::str::FromStr;
use tabled::builder::Builder;
use tabled::settings::Style;
use tempfile::tempfile;
use tokio::process::Command;
use tokio::runtime::Handle;
use tokio::time::{Duration, Instant, sleep, timeout};
use tracing_log::LogTracer;
use tracing_subscriber::{EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
mod adhoc;
mod bench;
mod cli;
mod debug;
mod shell;
mod tags;
mod util;
pub(crate) const UPGRADE_NOTICE: &str = "Try upgrading to the latest CLI version to resolve this issue. Also make sure the pipeline is recompiled with the latest version of feldera. Report it on github.com/feldera/feldera if the issue persists.";
use crate::adhoc::handle_adhoc_query;
use crate::cli::*;
use crate::shell::shell;
use crate::util::{is_sensitive_header, parse_header, terminal_safe};
fn unique_file(base: &str, extension: &str) -> Result<(PathBuf, File), std::io::Error> {
let mut path = PathBuf::from(format!("{}.{}", base, extension));
let mut count = 1;
loop {
let file = File::create_new(&path);
match file {
Ok(file) => {
return Ok((path, file));
}
Err(e) => {
if e.kind() == ErrorKind::AlreadyExists {
path = PathBuf::from(format!("{}_{}.{}", base, count, extension));
count += 1;
} else {
return Err(e);
}
}
}
}
}
fn make_auth_headers(auth: &Option<String>) -> Result<HeaderMap, InvalidHeaderValue> {
let mut headers = HeaderMap::new();
if let Some(key) = auth {
let mut header_value = HeaderValue::from_str(format!("Bearer {}", key).as_str())?;
header_value.set_sensitive(true);
headers.insert(reqwest::header::AUTHORIZATION, header_value);
}
Ok(headers)
}
pub(crate) struct ClientOpts {
pub host: String,
pub insecure: bool,
pub tls_cert: Option<std::path::PathBuf>,
pub auth: Option<String>,
pub oidc_token_file: Option<std::path::PathBuf>,
pub headers: Vec<String>,
pub timeout_secs: Option<u64>,
pub tenant: Option<String>,
pub retries: u32,
}
impl ClientOpts {
fn from_cli(cli: &Cli) -> Self {
Self {
host: cli.host.clone(),
insecure: cli.insecure,
tls_cert: cli.tls_cert.clone(),
auth: cli.auth.clone(),
oidc_token_file: cli.oidc_token_file.clone(),
headers: cli.headers.clone(),
timeout_secs: cli.timeout,
tenant: cli.tenant.clone(),
retries: cli.retries,
}
}
}
pub(crate) fn make_client(opts: ClientOpts) -> Result<Client, Box<dyn std::error::Error>> {
let ClientOpts {
host,
insecure,
tls_cert,
auth,
oidc_token_file,
headers: extra_headers,
timeout_secs,
tenant,
retries,
} = opts;
let mut client_builder = reqwest::ClientBuilder::new()
.http1_only()
.danger_accept_invalid_certs(insecure);
if let Some(timeout) = timeout_secs {
client_builder = client_builder.timeout(Duration::from_secs(timeout));
}
if let Some(cert_path) = tls_cert {
let pem = std::fs::read(&cert_path).map_err(|e| {
format!(
"Failed to read TLS certificate file `{}`: {e}",
cert_path.display()
)
})?;
let certs = reqwest::Certificate::from_pem_bundle(&pem).map_err(|e| {
format!(
"Failed to parse TLS certificate file `{}` as PEM: {e}",
cert_path.display()
)
})?;
if certs.is_empty() {
return Err(format!(
"TLS certificate file `{}` did not contain any PEM-encoded certificates",
cert_path.display()
)
.into());
}
for cert in certs {
client_builder = client_builder.add_root_certificate(cert);
}
}
let mut headers = HeaderMap::new();
if let Some(tenant) = &tenant {
headers.insert(
"Feldera-Tenant",
HeaderValue::from_str(tenant)
.map_err(|e| format!("Invalid --tenant value `{tenant}`: {e}"))?,
);
}
if host.starts_with("https://") {
let resolved_auth = match oidc_token_file {
Some(path) => Some(read_oidc_token_file(&path)?),
None => auth,
};
headers.extend(make_auth_headers(&resolved_auth)?);
} else if host.starts_with("http://") && (auth.is_some() || oidc_token_file.is_some()) {
warn!(
"The provided credentials are not added to the request because {host} does not use `https`."
);
}
for spec in &extra_headers {
let (name, value) = parse_header(spec)?;
if host.starts_with("http://") && is_sensitive_header(&name) {
warn!("Header `{name}` is sent in the clear because {host} does not use `https`.");
}
headers.insert(name, value);
}
client_builder = client_builder.default_headers(headers);
let client = client_builder.build()?;
let retry_policy = RetryPolicy {
max_retries: retries,
..RetryPolicy::default()
};
Ok(Client::new_with_client(host.as_str(), client, retry_policy))
}
fn read_oidc_token_file(path: &std::path::Path) -> Result<String, Box<dyn std::error::Error>> {
let token = std::fs::read_to_string(path)
.map_err(|e| format!("failed to read OIDC token file `{}`: {e}", path.display()))?
.trim()
.to_string();
if token.is_empty() {
return Err(format!("OIDC token file `{}` is empty", path.display()).into());
}
if let Some(offset) = token.find(|c: char| c.is_ascii_control() && c != '\t') {
let found = match token.as_bytes()[offset] {
b'\n' | b'\r' => "a line break",
_ => "a control character",
};
return Err(format!(
"OIDC token file `{}` does not hold a single-line token: found {found} at byte {offset}",
path.display()
)
.into());
}
Ok(token)
}
struct CacheDisabler {
name: String,
client: Client,
original_pc: ProgramConfig,
}
impl CacheDisabler {
async fn new(name: String, client: &Client, original_pc: ProgramConfig) -> Self {
Self {
name,
client: client.clone(),
original_pc,
}
}
async fn restore(&self) {
if self.original_pc.cache {
debug!("Restoring original cache flag for pipeline {}", self.name);
self.set_cache_flag(true).await;
} else {
debug!(
"Cache for pipeline stays disabled as it was previously disabled {}",
self.name
);
}
}
async fn disable(&self) {
self.set_cache_flag(false).await;
}
async fn set_cache_flag(&self, flag: bool) {
let mut pc = self.original_pc.clone();
pc.cache = flag;
self.client
.patch_pipeline()
.pipeline_name(self.name.clone())
.body(PatchPipeline {
description: None,
tags: None,
name: None,
program_code: None,
udf_rust: None,
udf_toml: None,
program_config: Some(pc),
runtime_config: None,
deployment_config: None,
})
.send()
.await
.map_err(handle_errors_fatal(
self.client.baseurl().to_string(),
format!("Failed to set compilation cache to {flag}").leak(),
1,
))
.unwrap();
}
}
fn handle_errors_fatal(
server: String,
msg: &'static str,
exit_code: i32,
) -> Box<dyn Fn(Error<ErrorResponse>) -> Infallible + Send> {
assert_ne!(exit_code, 0, "Exit code must not be 0");
Box::new(move |err: Error<ErrorResponse>| -> Infallible {
match err {
Error::ErrorResponse(e) => {
eprintln!("{}", terminal_safe(&e.message));
debug!("Details: {:#?}", e.details);
}
Error::InvalidRequest(s) => {
eprintln!("{}: ", msg);
error!("Invalid request ({})", s);
error!("{}", UPGRADE_NOTICE);
}
Error::CommunicationError(e) => {
eprint!("{}: ", msg);
eprintln!("{}.", e);
eprintln!("Check your network connection.");
}
Error::InvalidUpgrade(e) => {
eprint!("{}: ", msg);
eprintln!(
"Failed to upgrade connection ({}). Check your network connection.",
e
);
}
Error::ResponseBodyError(e) => {
eprintln!("{}: ", msg);
error!(
"Unable to read the detailed error returned from {server} ({})",
e
);
error!("{}", UPGRADE_NOTICE);
}
Error::InvalidResponsePayload(b, e) => {
eprintln!("{}", msg);
if !b.is_empty() {
error!("Unable to parse the detailed response returned from `{server}`");
debug!("Parse Error: {:?}", e.to_string());
debug!("Response payload: {:?}", String::from_utf8_lossy(&b));
} else {
error!("No detailed response (empty payload) returned from `{server}`");
}
error!("{}", UPGRADE_NOTICE);
}
Error::UnexpectedResponse(r) => {
let status = r.status();
let is_http = server.starts_with("http://");
std::io::stdout().flush().unwrap();
std::io::stderr().flush().unwrap();
let h = Handle::current();
let body = std::thread::spawn(move || h.block_on(r.text()).ok())
.join()
.unwrap();
let server_msg = body.as_deref().and_then(|b| {
serde_json::from_str::<serde_json::Value>(b)
.ok()
.and_then(|v| {
v.get("message")
.and_then(|m| m.as_str())
.map(str::to_string)
})
});
match server_msg {
Some(m) => {
eprintln!("{msg}: {}", terminal_safe(&m));
if status == StatusCode::UNAUTHORIZED && is_http {
eprintln!("Did you mean to use https?");
}
}
None if status == StatusCode::UNAUTHORIZED => {
eprintln!("{msg}: authentication failed.");
if is_http {
eprintln!(
"fda sends credentials only over https, so this request carried none. Did you mean to use https?"
);
} else {
eprintln!("Check that the API key or token is valid and not expired.");
}
}
None => {
warn!(
"Unexpected error response from {server} -- this can happen if you're running different fda and feldera versions."
);
warn!("{}", UPGRADE_NOTICE);
debug!(
"Received HTTP status `{status}` which is not declared as an expected response in OpenAPI."
);
match body {
Some(b) if !b.is_empty() => eprintln!("{msg}: {b}"),
_ => eprintln!("{msg}"),
}
}
}
}
Error::Custom(e) => {
eprintln!("{}: {}", msg, e);
}
};
std::process::exit(exit_code);
})
}
async fn api_key_commands(format: OutputFormat, action: ApiKeyActions, client: Client) {
match action {
ApiKeyActions::Create { name, role } => {
debug!("Creating API key: {}", name);
let role = match role {
ApiKeyRole::Read => feldera_rest_api::types::MintableKeyRole::Read,
ApiKeyRole::Write => feldera_rest_api::types::MintableKeyRole::Write,
};
let response = client
.post_api_key()
.body(NewApiKeyRequest {
name,
role: Some(role),
})
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to create API key",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!(
"API key '{}' created: {}",
terminal_safe(&response.name),
response.api_key
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize API key response")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
ApiKeyActions::Delete { name } => {
debug!("Deleting API key: {}", name);
client
.delete_api_key()
.api_key_name(name.as_str())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to delete API key",
1,
))
.unwrap();
println!("API key '{}' deleted", name);
}
ApiKeyActions::List => {
debug!("Listing API keys");
let response = client
.list_api_keys()
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to list API keys",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let mut rows = vec![];
rows.push(["name".to_string(), "role".to_string(), "id".to_string()]);
for key in response.iter() {
rows.push([
terminal_safe(&key.name),
key.role.to_string(),
key.id.0.to_string(),
]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize API key list")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
}
}
async fn oidc_trust_commands(format: OutputFormat, action: OidcTrustActions, client: Client) {
match action {
OidcTrustActions::Create {
name,
issuer,
subject,
audience,
description,
role,
} => {
debug!("Creating OIDC trust relationship: {name}");
let role = role.map(|r| match r {
TrustRole::Read => feldera_rest_api::types::MemberRole::Read,
TrustRole::Write => feldera_rest_api::types::MemberRole::Write,
TrustRole::Admin => feldera_rest_api::types::MemberRole::Admin,
});
let body = NewOidcTrustRequest::builder()
.name(name.clone())
.issuer(issuer)
.subject(subject)
.audience(audience)
.description(description)
.role(role);
let response = client
.post_oidc_trust()
.body(body)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to create OIDC trust relationship",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!(
"OIDC trust '{}' created (id: {})",
response.name, response.id.0
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize OIDC trust response")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
OidcTrustActions::Delete { name } => {
debug!("Deleting OIDC trust relationship: {name}");
client
.delete_oidc_trust()
.name(name.as_str())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to delete OIDC trust relationship",
1,
))
.unwrap();
println!("OIDC trust '{name}' deleted");
}
OidcTrustActions::List => {
debug!("Listing OIDC trust relationships");
let response = client
.list_oidc_trust()
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to list OIDC trust relationships",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let mut rows = vec![[
"name".to_string(),
"role".to_string(),
"issuer".to_string(),
"subject".to_string(),
"audience".to_string(),
"description".to_string(),
]];
for t in response.iter() {
rows.push([
terminal_safe(&t.name),
t.role.to_string(),
terminal_safe(&t.issuer),
terminal_safe(&t.subject),
terminal_safe(&t.audience.clone().unwrap_or_default()),
terminal_safe(&t.description.clone().unwrap_or_default()),
]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize OIDC trust list")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
}
}
async fn member_commands(format: OutputFormat, action: MemberActions, client: Client) {
match action {
MemberActions::List => {
debug!("Listing tenant members");
let response = client
.list_tenant_users()
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to list tenant members",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let mut rows = vec![[
"user id".to_string(),
"name".to_string(),
"subject".to_string(),
"email".to_string(),
"verified".to_string(),
"role".to_string(),
"origin".to_string(),
]];
for m in response.iter() {
rows.push([
m.user_id.0.to_string(),
terminal_safe(&m.display_name.clone().unwrap_or_default()),
terminal_safe(&m.subject),
terminal_safe(&m.email.clone().unwrap_or_default()),
if m.email_verified.unwrap_or(false) {
"yes"
} else {
"no"
}
.to_string(),
m.role.to_string(),
m.origin.as_ref().map(|o| o.to_string()).unwrap_or_default(),
]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize member list")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
MemberActions::Add {
subject,
role,
email,
} => {
debug!("Adding member {subject}");
let response = client
.add_tenant_user()
.body_map(|body| {
body.subject(subject.clone())
.role(role)
.email(email.clone())
})
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to add tenant member",
1,
))
.unwrap();
println!(
"Added '{subject}' as {role} (user id {}).",
response.user_id.0
);
}
MemberActions::SetRole { user_id, role } => {
debug!("Setting role for {user_id}");
client
.put_tenant_user()
.user_id(user_id)
.body_map(|body| body.role(role))
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to set member role",
1,
))
.unwrap();
println!("User {user_id} is now {role}.");
}
MemberActions::Remove { user_id } => {
debug!("Removing member {user_id}");
client
.delete_tenant_user()
.user_id(user_id)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to remove tenant member",
1,
))
.unwrap();
println!("Removed user {user_id} from the tenant.");
}
}
}
async fn tenant_commands(format: OutputFormat, action: TenantActions, client: Client) {
match action {
TenantActions::List => {
debug!("Listing tenants");
let response = client
.list_tenants()
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to list tenants",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let mut rows = vec![[
"name".to_string(),
"id".to_string(),
"initial provider".to_string(),
]];
for tenant in response.iter() {
rows.push([
terminal_safe(&tenant.name),
tenant.id.0.to_string(),
terminal_safe(&tenant.initial_provider),
]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize tenant list")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
TenantActions::Get { tenant } => {
debug!("Retrieving tenant {tenant}");
let response = client
.get_tenant()
.tenant_id(&tenant)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to retrieve tenant",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let rows = vec![
[
"name".to_string(),
"id".to_string(),
"initial provider".to_string(),
],
[
response.name.clone(),
response.id.0.to_string(),
response.initial_provider.clone(),
],
];
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize tenant")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
TenantActions::Create { name } => {
debug!("Creating tenant {name}");
let response = client
.create_tenant()
.body_map(|body| body.name(name.clone()))
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to create tenant",
1,
))
.unwrap();
let created = response.status() == reqwest::StatusCode::CREATED;
match format {
OutputFormat::Text => {
if created {
println!(
"Created tenant '{}' ({}).",
terminal_safe(&response.name),
response.id.0
);
} else {
println!(
"Tenant '{}' already exists ({}).",
response.name, response.id.0
);
}
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize tenant")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
TenantActions::Rename {
tenant_id,
name,
displace_existing,
} => {
debug!("Renaming tenant {tenant_id} to {name}");
let response = client
.patch_tenant()
.tenant_id(tenant_id)
.body_map(|body| body.name(name.clone()).displace_existing(displace_existing))
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to rename tenant",
1,
))
.unwrap();
match &response.displaced {
Some(displaced) => println!(
"Renamed tenant {tenant_id} to '{name}'. The tenant that held the name is now '{}' ({}).",
displaced.name, displaced.id.0
),
None => println!("Renamed tenant {tenant_id} to '{name}'."),
}
}
TenantActions::Delete { tenant_id } => {
debug!("Deleting tenant {tenant_id}");
client
.delete_tenant()
.tenant_id(tenant_id)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to delete tenant",
1,
))
.unwrap();
println!("Tenant {tenant_id} deleted.");
}
}
}
async fn pipelines(format: OutputFormat, client: Client) {
debug!("Listing pipelines");
let response = client
.list_pipelines()
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to list pipelines",
1,
))
.unwrap();
let mut rows = vec![];
rows.push(["name".to_string(), "status".to_string()]);
for pipeline in response.iter() {
rows.push([
terminal_safe(&pipeline.name),
pipeline.deployment_status.to_string(),
]);
}
match format {
OutputFormat::Text => {
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline list")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
fn key_name(key: RuntimeConfigKey) -> String {
key.to_possible_value()
.map(|value| value.get_name().to_string())
.unwrap_or_else(|| format!("{key:?}"))
}
fn parse_value<T>(key: RuntimeConfigKey, value: &str) -> Result<T, String>
where
T: FromStr,
T::Err: Display,
{
value.parse::<T>().map_err(|error| {
format!(
"`{value}` is not a valid value for `{}`: {error}",
key_name(key)
)
})
}
fn parse_size_with_unit(key: RuntimeConfigKey, value: &str) -> Result<ConfigSize, String> {
match ConfigSize::from_str_with_unit(value) {
Ok((size, true)) => Ok(size),
Ok((_, false)) => Err(format!(
"`{value}` is not a valid value for `{}`: write a size with a unit, such as `{value}M`",
key_name(key)
)),
Err(error) => Err(format!(
"`{value}` is not a valid value for `{}`: {error}",
key_name(key)
)),
}
}
fn patch_runtime_config(
rc: &mut RuntimeConfig,
key: RuntimeConfigKey,
value: &str,
) -> Result<(), String> {
match key {
RuntimeConfigKey::Workers => {
rc.workers = parse_value(key, value)?;
}
RuntimeConfigKey::Storage => {
let enable: bool = parse_value(key, value)?;
if enable != rc.storage.is_some() {
rc.storage = enable.then(StorageOptions::default);
}
}
RuntimeConfigKey::FaultTolerance => {
rc.fault_tolerance.model = match value {
"false" | "none" => None,
"true" => Some(FtModel::default()),
_ => Some(parse_value(key, value)?),
}
}
RuntimeConfigKey::CheckpointInterval => {
rc.fault_tolerance.checkpoint_interval = Some(match value {
"none" | "null" => None,
_ => match value.parse::<u64>() {
Ok(0) => {
eprintln!(
"a bare 0 is deprecated; write `none` to disable periodic checkpoints"
);
None
}
Ok(secs) => {
eprintln!(
"a bare number of seconds is deprecated; write `{secs}s`, \
or `none` to disable periodic checkpoints"
);
Some(ConfigDuration::from_secs(secs))
}
Err(_) => Some(parse_value(key, value)?),
},
});
}
RuntimeConfigKey::CpuProfiler => {
rc.cpu_profiler = parse_value(key, value)?;
}
RuntimeConfigKey::Tracing => {
rc.tracing = parse_value(key, value)?;
}
RuntimeConfigKey::TracingEndpointJaeger => {
rc.tracing_endpoint_jaeger = parse_value(key, value)?;
}
RuntimeConfigKey::MinBatchSizeRecords => {
rc.min_batch_size_records = parse_value(key, value)?;
}
RuntimeConfigKey::MaxBufferingDelayUsecs => {
let legacy = parse_value(key, value)?;
eprintln!(
"`max_buffering_delay_usecs` is deprecated; use `max_buffering_delay` instead"
);
rc.max_buffering_delay = Some(ConfigDuration::from_micros(legacy));
}
RuntimeConfigKey::MaxBufferingDelay => {
rc.max_buffering_delay = Some(parse_value(key, value)?);
}
RuntimeConfigKey::CpuCoresMin => {
rc.resources.cpu_cores_min = Some(parse_value(key, value)?);
}
RuntimeConfigKey::CpuCoresMax => {
rc.resources.cpu_cores_max = Some(parse_value(key, value)?);
}
RuntimeConfigKey::MemoryMbMin => {
eprintln!("`memory_mb_min` is deprecated; use `memory_min` instead");
rc.resources.memory_min = Some(ConfigSize::from_mb(parse_value(key, value)?));
}
RuntimeConfigKey::MemoryMbMax => {
eprintln!("`memory_mb_max` is deprecated; use `memory_max` instead");
rc.resources.memory_max = Some(ConfigSize::from_mb(parse_value(key, value)?));
}
RuntimeConfigKey::MemoryMin => {
rc.resources.memory_min = Some(parse_size_with_unit(key, value)?);
}
RuntimeConfigKey::MemoryMax => {
rc.resources.memory_max = Some(parse_size_with_unit(key, value)?);
}
RuntimeConfigKey::StorageMbMax => {
eprintln!("`storage_mb_max` is deprecated; use `storage_max` instead");
rc.resources.storage_max = Some(ConfigSize::from_mb(parse_value(key, value)?));
}
RuntimeConfigKey::StorageMin => {
rc.resources.storage_min = Some(parse_size_with_unit(key, value)?);
}
RuntimeConfigKey::StorageMax => {
rc.resources.storage_max = Some(parse_size_with_unit(key, value)?);
}
RuntimeConfigKey::MaxRss => {
rc.max_rss = Some(parse_size_with_unit(key, value)?);
}
RuntimeConfigKey::DatafusionMemory => {
rc.datafusion_memory = Some(parse_size_with_unit(key, value)?);
}
RuntimeConfigKey::StorageClass => {
rc.resources.storage_class = Some(parse_value(key, value)?);
}
RuntimeConfigKey::VolumeAttributesClass => {
rc.resources.volume_attributes_class = Some(parse_value(key, value)?);
}
RuntimeConfigKey::MinStorageBytes => {
eprintln!("`min_storage_bytes` is deprecated; use `min_spill_size` instead");
if let Some(storage) = rc.storage.as_mut() {
storage.min_spill_size = Some(ConfigSize::from_bytes(parse_value(key, value)?));
}
}
RuntimeConfigKey::MinSpillSize => {
if let Some(storage) = rc.storage.as_mut() {
storage.min_spill_size = Some(parse_value(key, value)?);
}
}
RuntimeConfigKey::ClockResolutionUsecs => {
let legacy = parse_value(key, value)?;
eprintln!("`clock_resolution_usecs` is deprecated; use `clock_resolution` instead");
rc.clock_resolution = Some(ConfigDuration::from_micros(legacy));
}
RuntimeConfigKey::ClockResolution => {
rc.clock_resolution = Some(parse_value(key, value)?);
}
RuntimeConfigKey::Logging => {
rc.logging = Some(parse_value(key, value)?);
}
RuntimeConfigKey::HttpWorkers => {
rc.http_workers = Some(parse_value(key, value)?);
}
RuntimeConfigKey::IoWorkers => {
rc.io_workers = Some(parse_value(key, value)?);
}
RuntimeConfigKey::DevTweaks => {
rc.dev_tweaks = serde_json::from_str(value).map_err(|error| {
format!(
"`{value}` is not valid JSON for `{}`: {error}",
key_name(key)
)
})?;
}
};
Ok(())
}
async fn read_file(file_path: Option<String>) -> Result<Option<String>, ()> {
if let Some(path) = file_path {
match tokio::fs::read_to_string(path.as_str()).await {
Ok(udf_code) => {
debug!("Read from file: {}", path);
Ok(Some(udf_code))
}
Err(e) => {
eprintln!("Failed to read '{}': {}", path, e);
Err(())
}
}
} else {
Ok(None)
}
}
async fn read_program_code(
program_path: Option<String>,
stdin: bool,
) -> Result<Option<String>, ()> {
if stdin {
let mut program_code = String::new();
let mut stdin = std::io::stdin();
if stdin.read_to_string(&mut program_code).is_ok() {
debug!("Read program code from stdin");
Ok(Some(program_code))
} else {
eprintln!("Failed to read program code from stdin");
Err(())
}
} else {
read_file(program_path).await
}
}
async fn validate_program(
program_path: Option<String>,
stdin: bool,
runtime_version: Option<String>,
ir: bool,
client: Client,
) {
let program_code = match read_program_code(program_path, stdin).await {
Ok(Some(program_code)) => program_code,
Ok(None) => {
eprintln!("No SQL program provided: pass a program file path or --stdin.");
std::process::exit(1);
}
Err(()) => std::process::exit(1),
};
let response = client
.post_validate_program()
.body(ValidateProgramRequest {
program_code,
runtime_version,
ir: Some(ir),
})
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to validate program",
1,
))
.unwrap();
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("failed to serialize validation response")
);
}
async fn wait_for_status(
client: &Client,
name: String,
wait_for: CombinedStatus,
waiting_text: &str,
) {
wait_for_status_one_of(client, name, &[wait_for], waiting_text).await;
}
async fn wait_for_status_one_of(
client: &Client,
name: String,
wait_for: &[CombinedStatus],
waiting_text: &str,
) -> CombinedStatus {
let mut print_every_30_seconds = Instant::now();
loop {
let pc = client
.get_pipeline()
.pipeline_name(name.clone())
.selector(PipelineFieldSelector::Status)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get program config",
1,
))
.unwrap();
if wait_for.contains(&pc.deployment_status) {
return pc.deployment_status;
}
if print_every_30_seconds.elapsed().as_secs() > 30 {
info!(
"{} (current status: {})",
waiting_text, pc.deployment_status
);
print_every_30_seconds = Instant::now();
}
if let Some(deployment_error) = &pc.deployment_error {
if deployment_error.error_code == "StartFailedDueToFailedCompilation" {
eprintln!("Pipeline failed to start due to the following compilation error:");
eprintln!();
eprintln!(
"{}",
deployment_error
.details
.as_object()
.unwrap_or(&serde_json::Map::new())
.get("compiler_error")
.and_then(|e| e.as_str())
.unwrap_or_default()
);
} else {
eprintln!("{}", terminal_safe(&deployment_error.message));
}
std::process::exit(1);
}
sleep(Duration::from_millis(500)).await;
}
}
async fn wait_for_storage_status(
client: &Client,
name: String,
wait_for: StorageStatus,
waiting_text: &str,
) {
let mut print_every_30_seconds = Instant::now();
let mut is_transitioning = true;
while is_transitioning {
let pc = client
.get_pipeline()
.pipeline_name(name.clone())
.selector(PipelineFieldSelector::Status)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get program config",
1,
))
.unwrap();
is_transitioning = pc.storage_status != wait_for;
if print_every_30_seconds.elapsed().as_secs() > 30 {
info!("{}", waiting_text);
print_every_30_seconds = Instant::now();
}
sleep(Duration::from_millis(500)).await;
}
}
async fn wait_for_checkpoint(
client: &Client,
name: String,
mut checkpoint: CheckpointResponse,
waiting_text: &str,
) -> CheckpointResponse {
let mut print_every_30_seconds = Instant::now();
let mut is_waiting_for_checkpoint = true;
while is_waiting_for_checkpoint {
let mut request = client.get_checkpoint_status().pipeline_name(name.clone());
if let Some(uuid) = checkpoint.incarnation_uuid {
request = request.incarnation_uuid(uuid);
}
let cs = match request.send().await {
Ok(cs) => cs,
Err(Error::ErrorResponse(e)) if e.error_code == "IncarnationUuidMismatch" => {
sleep(Duration::from_millis(100)).await;
checkpoint = client
.checkpoint_pipeline()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to initiate pipeline checkpoint",
1,
))
.unwrap()
.into_inner();
continue;
}
Err(err) => Err(err)
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline checkpoint status",
1,
))
.unwrap(),
};
debug!("Checkpoint status {:?}", cs);
is_waiting_for_checkpoint = cs
.success
.map(|c| c < checkpoint.checkpoint_sequence_number)
.unwrap_or(true);
if !is_waiting_for_checkpoint {
break;
}
if let Some(failure) = &cs.failure {
if failure.sequence_number < checkpoint.checkpoint_sequence_number {
continue;
}
eprintln!(
"Pipeline failed to take the checkpoint#{} due to the following error:",
failure.sequence_number
);
eprintln!();
eprintln!("{}", failure.error);
std::process::exit(1);
}
if print_every_30_seconds.elapsed().as_secs() > 30 {
info!("{}", waiting_text);
print_every_30_seconds = Instant::now();
}
sleep(Duration::from_millis(500)).await;
}
checkpoint
}
async fn get_pipeline_tags(client: &Client, name: &str) -> Vec<String> {
client
.get_pipeline()
.pipeline_name(name)
.selector(PipelineFieldSelector::Status)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline tags",
1,
))
.unwrap()
.into_inner()
.tags
}
async fn get_known_tags(client: &Client) -> Vec<String> {
client
.list_pipelines()
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to list pipeline tags",
1,
))
.unwrap()
.into_inner()
.into_iter()
.flat_map(|pipeline| pipeline.tags)
.collect()
}
async fn set_pipeline_tags(client: &Client, name: &str, tags: Vec<String>) -> Vec<String> {
client
.patch_pipeline()
.pipeline_name(name)
.body(PatchPipeline {
description: None,
tags: Some(tags),
name: None,
program_code: None,
udf_rust: None,
udf_toml: None,
program_config: None,
runtime_config: None,
deployment_config: None,
})
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to update pipeline tags",
1,
))
.unwrap()
.into_inner()
.tags
}
fn print_pipeline_tags(format: OutputFormat, tags: &[String]) {
match format {
OutputFormat::Text => {
let names: Vec<&str> = tags.iter().map(|tag| tags::tag_display_name(tag)).collect();
println!("{}", names.join(","));
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(tags).expect("Failed to serialize tags")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
async fn pipeline(format: OutputFormat, action: PipelineAction, client: Client) {
match action {
PipelineAction::Create {
name,
program_path,
runtime_version,
use_platform_compiler,
profile,
tags,
udf_rs,
udf_toml,
stdin,
} => {
if let (Ok(program_code), Ok(udf_rust), Ok(udf_toml)) = (
read_program_code(program_path, stdin).await,
read_file(udf_rs).await,
read_file(udf_toml).await,
) {
let tags = if tags.is_empty() {
Vec::new()
} else {
tags::set_tags(tags, &get_known_tags(&client).await)
};
let response = client
.post_pipeline()
.body(PostPutPipeline {
description: None,
tags,
name: name.to_string(),
program_code: program_code.unwrap_or_default(),
udf_rust,
udf_toml,
program_config: Some(ProgramConfig {
cache: true,
profile: Some(profile),
runtime_version,
use_platform_compiler,
}),
runtime_config: None,
deployment_config: None,
})
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to create pipeline",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!("Pipeline created successfully.");
debug!("{:#?}", response);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline response")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
} else {
std::process::exit(1);
}
}
PipelineAction::Copy {
source,
destination,
} => {
let source_pipeline = client
.get_pipeline()
.pipeline_name(source.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get source pipeline",
1,
))
.unwrap()
.into_inner();
let Some(program_code) = source_pipeline.program_code else {
eprintln!(
"Source pipeline response did not include program code. {}",
UPGRADE_NOTICE
);
std::process::exit(1);
};
let Some(runtime_config) = source_pipeline.runtime_config else {
eprintln!(
"Source pipeline response did not include runtime configuration. {}",
UPGRADE_NOTICE
);
std::process::exit(1);
};
let Some(program_config) = source_pipeline.program_config else {
eprintln!(
"Source pipeline response did not include compilation configuration. {}",
UPGRADE_NOTICE
);
std::process::exit(1);
};
let response = client
.post_pipeline()
.body(PostPutPipeline {
description: source_pipeline.description,
tags: source_pipeline.tags,
name: destination.clone(),
program_code,
udf_rust: source_pipeline.udf_rust,
udf_toml: source_pipeline.udf_toml,
program_config: Some(program_config),
runtime_config: Some(runtime_config),
deployment_config: None,
})
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to copy pipeline",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!("Pipeline copied successfully.");
debug!("{:#?}", response);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline response")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
PipelineAction::Start {
name,
recompile,
no_wait,
initial,
bootstrap_policy,
silent_bootstrap,
concurrent_bootstrap,
no_dismiss_error,
} => {
if initial != "standby" && initial != "paused" && initial != "running" {
eprintln!("Unsupported `--initial`: {}", initial);
std::process::exit(1);
}
if bootstrap_policy != "allow"
&& bootstrap_policy != "reject"
&& bootstrap_policy != "await_approval"
{
eprintln!("Unsupported `--bootstrap-policy`: {}", bootstrap_policy);
std::process::exit(1);
}
let pc = client
.get_pipeline()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get program config",
1,
))
.unwrap();
if recompile {
let cd =
CacheDisabler::new(name.clone(), &client, pc.program_config.clone().unwrap())
.await;
cd.disable().await;
let new_program = if pc
.program_code
.as_ref()
.unwrap()
.ends_with(|c: char| c.is_whitespace())
{
pc.program_code.as_ref().unwrap().trim().to_string()
} else {
pc.program_code.clone().unwrap() + " "
};
client
.patch_pipeline()
.pipeline_name(name.clone())
.body(PatchPipeline {
description: None,
tags: None,
name: None,
program_code: Some(new_program),
udf_rust: None,
udf_toml: None,
program_config: None,
runtime_config: None,
deployment_config: None,
})
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to recompile pipeline",
1,
))
.unwrap();
info!("Forcing recompilation this may take a few seconds...");
cd.restore().await;
};
let mut print_every_30_seconds = Instant::now();
let mut compiling = true;
while compiling {
let pc = client
.get_pipeline()
.pipeline_name(name.clone())
.selector(PipelineFieldSelector::Status)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get program config",
1,
))
.unwrap();
compiling = matches!(pc.program_status, ProgramStatus::Pending)
|| matches!(pc.program_status, ProgramStatus::CompilingRust)
|| matches!(pc.program_status, ProgramStatus::CompilingSql)
|| matches!(pc.program_status, ProgramStatus::SqlCompiled);
if print_every_30_seconds.elapsed().as_secs() > 30 {
info!("Compiling pipeline...");
print_every_30_seconds = Instant::now();
}
sleep(Duration::from_millis(500)).await;
}
if !no_dismiss_error {
let response = client
.post_pipeline_dismiss_error()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to dismiss pipeline deployment error",
1,
))
.unwrap();
trace!("{:#?}", response);
}
let response = client
.post_pipeline_start()
.pipeline_name(name.clone())
.initial(&initial)
.bootstrap_policy(&bootstrap_policy)
.silent_bootstrap(silent_bootstrap)
.concurrent_bootstrap(concurrent_bootstrap)
.dismiss_error(false) .send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to start pipeline",
1,
))
.unwrap();
if !no_wait {
let status = wait_for_status_one_of(
&client,
name.clone(),
if initial == "running" {
&[CombinedStatus::Running, CombinedStatus::AwaitingApproval]
} else if initial == "paused" {
&[CombinedStatus::Paused, CombinedStatus::AwaitingApproval]
} else {
&[CombinedStatus::Standby, CombinedStatus::AwaitingApproval]
},
"Starting the pipeline...",
)
.await;
if status == CombinedStatus::AwaitingApproval {
let runtime_status_details = client
.get_pipeline()
.pipeline_name(name.clone())
.selector(PipelineFieldSelector::Status)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline status",
1,
))
.unwrap()
.deployment_runtime_status_details
.clone();
println!(
"Pipeline definition has changed since the last checkpoint. The pipeline is awaiting approval to proceed with bootstrapping the modified components. Run 'fda approve' to approve the changes or 'fda stop' to terminate the pipeline. Summary of changes:"
);
let diff_str = match runtime_status_details.map(|details| details.approval_diff)
{
None | Some(None) => "<not available>".to_string(),
Some(Some(diff)) => match format {
OutputFormat::Text => json_to_table(&diff)
.collapse()
.into_pool_table()
.to_string(),
OutputFormat::Json => serde_json::to_string_pretty(&diff)
.expect("Failed to serialize pipeline diff"),
_ => {
format!("<unsupported output format: {}>", format)
}
},
};
println!("{}", diff_str);
} else {
println!("Pipeline started successfully.");
}
}
trace!("{:#?}", response);
}
PipelineAction::Checkpoint { name, no_wait } => {
let response = client
.checkpoint_pipeline()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to initiate pipeline checkpoint",
1,
))
.unwrap();
trace!("{:#?}", response);
if !no_wait {
let completed = wait_for_checkpoint(
&client,
name.clone(),
response.into_inner(),
"Taking a checkpoint...",
)
.await;
println!(
"Pipeline checkpoint (#{}) taken successfully.",
completed.checkpoint_sequence_number
);
}
}
PipelineAction::Approve {
name,
silent_bootstrap,
concurrent_bootstrap,
} => {
let response = client
.post_pipeline_approve()
.pipeline_name(name.clone())
.silent_bootstrap(silent_bootstrap)
.concurrent_bootstrap(concurrent_bootstrap)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to approve pipeline changes",
1,
))
.unwrap();
println!("Pipeline changes approved successfully.");
trace!("{:#?}", response);
}
PipelineAction::Pause { name, no_wait } => {
let response = client
.post_pipeline_pause()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to pause pipeline",
1,
))
.unwrap();
if !no_wait {
wait_for_status(
&client,
name.clone(),
CombinedStatus::Paused,
"Pausing the pipeline...",
)
.await;
println!("Pipeline paused successfully.");
}
trace!("{:#?}", response);
}
PipelineAction::Resume { name, no_wait } => {
let response = client
.post_pipeline_resume()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to resume pipeline",
1,
))
.unwrap();
if !no_wait {
wait_for_status(
&client,
name.clone(),
CombinedStatus::Running,
"Resuming the pipeline...",
)
.await;
println!("Pipeline resumed successfully.");
}
trace!("{:#?}", response);
}
PipelineAction::Restart {
name,
recompile,
checkpoint,
no_wait,
initial,
bootstrap_policy,
silent_bootstrap,
concurrent_bootstrap,
no_dismiss_error,
} => {
let current_status = client
.get_pipeline()
.pipeline_name(name.clone())
.selector(PipelineFieldSelector::Status)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline status",
1,
))
.unwrap();
let _r = client
.post_pipeline_stop()
.pipeline_name(name.clone())
.force(!checkpoint)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to stop pipeline",
1,
))
.unwrap();
wait_for_status(
&client,
name.clone(),
CombinedStatus::Stopped,
"Stopping the pipeline...",
)
.await;
if current_status.deployment_status != CombinedStatus::Stopped {
println!("Pipeline stop successful.");
}
let _ = Box::pin(pipeline(
format,
PipelineAction::Start {
name,
recompile,
no_wait,
initial,
bootstrap_policy,
silent_bootstrap,
concurrent_bootstrap,
no_dismiss_error,
},
client,
))
.await;
}
PipelineAction::Stop {
name,
no_wait,
checkpoint,
} => {
let response = client
.post_pipeline_stop()
.pipeline_name(name.clone())
.force(!checkpoint)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to stop pipeline",
1,
))
.unwrap();
if !no_wait {
wait_for_status(
&client,
name.clone(),
CombinedStatus::Stopped,
"Shutting down the pipeline...",
)
.await;
println!("Pipeline stopped successfully.");
}
trace!("{:#?}", response);
}
PipelineAction::Clear { name, no_wait } => {
let response = client
.post_pipeline_clear()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to clear pipeline",
1,
))
.unwrap();
trace!("{:#?}", response);
if !no_wait {
wait_for_storage_status(
&client,
name.clone(),
StorageStatus::Cleared,
"Clearing pipeline...",
)
.await;
println!("Pipeline cleared successfully.");
}
}
PipelineAction::Stats { name } => {
let response = client
.get_pipeline_stats()
.pipeline_name(name)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline stats",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let stats_value = serde_json::to_value(response.as_ref())
.expect("Failed to serialize pipeline stats");
let table = json_to_table(&stats_value)
.collapse()
.into_pool_table()
.to_string();
println!("{}", table);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(response.as_ref())
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
PipelineAction::Metrics { name } => {
let format = match format {
OutputFormat::Json => MetricsFormat::Json,
OutputFormat::Prometheus => MetricsFormat::Prometheus,
_ => {
eprintln!(
"`{format}` is not supported as a metrics format (use `--format json` or `--format prometheus`)"
);
std::process::exit(1);
}
};
let response = client
.get_pipeline_metrics()
.format(format)
.pipeline_name(name)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline metrics",
1,
))
.unwrap();
let mut byte_stream = response.into_inner();
while let Some(chunk) = byte_stream.next().await {
match chunk {
Ok(chunk) => match stdout().write_all(&chunk) {
Ok(_) => (),
Err(error) => {
eprintln!("write to stdout failed ({error})");
std::process::exit(1);
}
},
Err(e) => {
eprintln!("ERROR: Unable to read server response: {}", e);
std::process::exit(1);
}
}
}
}
PipelineAction::Logs { name, watch } => {
let mut first_line = true;
let next_line_timeout = if !watch {
Duration::from_secs(2)
} else {
Duration::MAX
};
let mut request = client.get_pipeline_logs().pipeline_name(name);
if !watch {
request = request.follow(false);
}
let response = request.send().await;
match response {
Ok(response) => {
let mut byte_stream = response.into_inner();
while let Ok(Some(chunk)) = timeout(
if first_line {
Duration::from_secs(5)
} else {
next_line_timeout
},
byte_stream.next(),
)
.await
{
match chunk {
Ok(chunk) => {
let text = String::from_utf8_lossy(&chunk);
print!("{}", text);
first_line = false;
}
Err(e) => {
eprintln!("ERROR: Unable to read server response: {}", e);
std::process::exit(1);
}
}
}
}
Err(e) => {
eprintln!("ERROR: Failed to get pipeline logs: {}", e);
std::process::exit(1);
}
}
}
PipelineAction::Delete { name, force } => {
if force {
let _ = Box::pin(pipeline(
format,
PipelineAction::Clear {
name: name.clone(),
no_wait: false,
},
client.clone(),
))
.await;
}
let response = client
.delete_pipeline()
.pipeline_name(name)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to delete the pipeline",
1,
))
.unwrap();
println!("Pipeline deleted successfully.");
trace!("{:#?}", response);
}
PipelineAction::Status { name } => {
let response = client
.get_pipeline()
.pipeline_name(name)
.selector(PipelineFieldSelector::Status)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline status",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let value = serde_json::to_value(json! {
{
"deployment_status": response.deployment_status,
"deployment_status_since": response.deployment_status_since,
"deployment_desired_status": response.deployment_desired_status.to_string(),
"deployment_error": response.deployment_error,
"program_status": response.program_status,
"program_status_since": response.program_status_since,
}
})
.unwrap();
let table = json_to_table(&value).collapse().to_string();
println!("{}", table);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
PipelineAction::Config { name } => {
let response = client
.get_pipeline()
.pipeline_name(name)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline config",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let value = serde_json::to_value(&response.runtime_config).unwrap();
let table = json_to_table(&value).collapse().to_string();
println!("{}", table);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.runtime_config)
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
PipelineAction::SetConfig { name, key, value } => {
let mut rc = client
.get_pipeline()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline config",
1,
))
.map(|response| response.runtime_config.clone())
.unwrap()
.unwrap();
if let Err(error) = patch_runtime_config(&mut rc, key, value.as_str()) {
eprintln!("{error}");
std::process::exit(1);
}
let response = client
.patch_pipeline()
.pipeline_name(name)
.body(PatchPipeline {
description: None,
tags: None,
name: None,
program_code: None,
udf_rust: None,
udf_toml: None,
program_config: None,
runtime_config: Some(rc),
deployment_config: None,
})
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to set runtime config",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!("Runtime config updated successfully.");
let value = serde_json::to_value(&response.runtime_config).unwrap();
let table = json_to_table(&value).collapse().to_string();
println!("{}", table);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.runtime_config)
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
PipelineAction::Tags { name } => {
let current = get_pipeline_tags(&client, &name).await;
print_pipeline_tags(format, ¤t);
}
PipelineAction::SetTags { name, tags } => {
let requested = tags::split_tag_list(&tags);
let known = get_known_tags(&client).await;
let updated =
set_pipeline_tags(&client, &name, tags::set_tags(requested, &known)).await;
if matches!(format, OutputFormat::Text) {
println!("Tags updated successfully.");
}
print_pipeline_tags(format, &updated);
}
PipelineAction::UpdateRuntime { name } => {
let response = client
.post_update_runtime()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to update pipeline runtime",
1,
))
.unwrap();
println!("Pipeline runtime updated successfully.");
trace!("{:#?}", response);
}
PipelineAction::Program { action } => program(format, action, client).await,
PipelineAction::Connector {
name,
relation_name,
connector_name,
action,
} => {
connector(
format,
name,
relation_name.as_str(),
connector_name.as_str(),
action,
client,
)
.await
}
PipelineAction::HeapProfile {
name,
pprof,
output,
} => {
let response = client
.get_pipeline_heap_profile()
.pipeline_name(name)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to obtain heap profile",
1,
))
.unwrap();
let mut byte_stream = response.into_inner();
let mut buffer = Vec::new();
while let Some(chunk) = byte_stream.next().await {
match chunk {
Ok(chunk) => buffer.extend_from_slice(&chunk),
Err(e) => {
eprintln!("ERROR: Unable to read server response: {}", e);
std::process::exit(1);
}
}
}
match output {
None => {
let mut tempfile = tempfile().unwrap_or_else(|e| {
eprintln!("ERROR: Failed to create temporary file: {e}");
std::process::exit(1);
});
tempfile.write_all(&buffer).unwrap();
let mut args = pprof.split_whitespace();
let command_name = args.next().expect("`pprof` command cannot be empty");
let mut command = Command::new(command_name);
command.args(args);
command.arg("/dev/stdin");
command.stdin(tempfile);
match command.status().await {
Err(e) => {
eprintln!(
"ERROR: Failed to execute `{command_name}` ({e}). `pprof` is probably not installed. Install it from <https://github.com/google/pprof>."
);
std::process::exit(1);
}
Ok(exit_status) if !exit_status.success() => {
eprintln!("Child process exited with {exit_status}");
std::process::exit(1);
}
Ok(_success) => (),
}
}
Some(filename) => {
tokio::fs::write(&filename, buffer)
.await
.unwrap_or_else(|e| {
eprintln!("ERROR: Failed to write {}: {e}", filename.display());
std::process::exit(1);
});
}
}
}
PipelineAction::CircuitProfile { name, output } => {
let response = client
.get_pipeline_circuit_profile()
.pipeline_name(&name)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to obtain circuit profile",
1,
))
.unwrap();
let mut byte_stream = response.into_inner();
let mut buffer = Vec::new();
while let Some(chunk) = byte_stream.next().await {
match chunk {
Ok(chunk) => buffer.extend_from_slice(&chunk),
Err(e) => {
eprintln!("ERROR: Unable to read server response: {}", e);
std::process::exit(1);
}
}
}
match output {
None => {
let (path, mut file) =
unique_file(format!("{name}-circuit-profile").as_str(), "zip")
.unwrap_or_else(|e| {
eprintln!("ERROR: Failed to create circuit profile file: {e}");
std::process::exit(1);
});
file.write_all(&buffer).unwrap_or_else(|e| {
eprintln!("ERROR: Failed to create temporary file: {e}");
std::process::exit(1);
});
println!("Circuit profile written to {}", path.display());
}
Some(filename) => {
tokio::fs::write(&filename, buffer)
.await
.unwrap_or_else(|e| {
eprintln!("ERROR: Failed to write {}: {e}", filename.display());
std::process::exit(1);
});
println!("Circuit profile written to {}", filename.display());
}
}
}
PipelineAction::SupportBundle {
name,
output,
limit,
no_collect,
no_circuit_profile,
no_heap_profile,
no_metrics,
no_logs,
no_stats,
no_pipeline_config,
no_system_config,
no_dataflow_graph,
no_pipeline_events,
} => {
let mut request = client.get_pipeline_support_bundle().pipeline_name(&name);
if let Some(limit) = limit {
request = request.limit(limit);
}
if no_collect {
request = request.collect(false);
}
if no_circuit_profile {
request = request.circuit_profile(false);
}
if no_heap_profile {
request = request.heap_profile(false);
}
if no_metrics {
request = request.metrics(false);
}
if no_logs {
request = request.logs(false);
}
if no_stats {
request = request.stats(false);
}
if no_pipeline_config {
request = request.pipeline_config(false);
}
if no_system_config {
request = request.system_config(false);
}
if no_dataflow_graph {
request = request.dataflow_graph(false);
}
if no_pipeline_events {
request = request.pipeline_events(false);
}
let response = request
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to obtain support bundle",
1,
))
.unwrap();
let mut byte_stream = response.into_inner();
let mut buffer = Vec::new();
while let Some(chunk) = byte_stream.next().await {
match chunk {
Ok(chunk) => buffer.extend_from_slice(&chunk),
Err(e) => {
eprintln!("ERROR: Unable to read server response: {}", e);
std::process::exit(1);
}
}
}
match output {
None => {
let (path, mut file) =
unique_file(format!("{name}-support-bundle").as_str(), "zip")
.unwrap_or_else(|e| {
eprintln!("ERROR: Failed to create circuit profile file: {e}");
std::process::exit(1);
});
file.write_all(&buffer).unwrap_or_else(|e| {
eprintln!("ERROR: Failed to create temporary file: {e}");
std::process::exit(1);
});
println!("Circuit profile written to {}", path.display());
}
Some(filename) => {
tokio::fs::write(&filename, buffer)
.await
.unwrap_or_else(|e| {
eprintln!("ERROR: Failed to write {}: {e}", filename.display());
std::process::exit(1);
});
println!("Support bundle written to {}", filename.display());
}
}
}
PipelineAction::Shell {
name,
start,
initial,
bootstrap_policy,
silent_bootstrap,
concurrent_bootstrap,
no_dismiss_error,
} => {
let client2 = client.clone();
if start {
let _ = Box::pin(pipeline(
format,
PipelineAction::Start {
name: name.clone(),
recompile: false,
no_wait: false,
initial,
bootstrap_policy,
silent_bootstrap,
concurrent_bootstrap,
no_dismiss_error,
},
client,
))
.await;
}
shell(format, name, client2).await
}
PipelineAction::Query { name, sql, stdin } => {
handle_adhoc_query(client, format, name, sql, stdin).await
}
PipelineAction::CompletionToken {
name,
table,
connector,
} => {
let response = client
.completion_token()
.pipeline_name(name)
.table_name(table)
.connector_name(connector)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get completion status",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!("{}", response.into_inner().token);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!(
"Unsupported output format: {}, falling back to text",
format
);
println!("{}", response.into_inner().token);
std::process::exit(1);
}
}
}
PipelineAction::CompletionStatus { name, token } => {
let response = client
.completion_status()
.pipeline_name(name)
.token(token)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get completion status",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!("{}", response.into_inner().status);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!(
"Unsupported output format, falling back to text: {}",
format
);
println!("{}", response.into_inner().status);
std::process::exit(1);
}
}
}
PipelineAction::StartTransaction { name } => {
let response = client
.start_transaction()
.pipeline_name(name)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to start transaction",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!(
"Transaction started successfully with ID: {}",
response.into_inner().transaction_id
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize transaction response")
);
}
_ => {
eprintln!(
"Unsupported output format, falling back to text: {}",
format
);
println!(
"Transaction started successfully with ID: {}",
response.into_inner().transaction_id
);
std::process::exit(1);
}
}
}
PipelineAction::CommitTransaction {
name,
transaction_id,
no_wait,
} => {
let stats = client
.get_pipeline_stats()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline stats",
1,
))
.unwrap();
let current_transaction_id = stats.into_inner().global_metrics.transaction_id;
if current_transaction_id == 0 {
eprintln!(
"Attempting to commit a transaction, but there is no transaction in progress"
);
std::process::exit(1);
}
if let Some(expected_transaction_id) = transaction_id
&& current_transaction_id != expected_transaction_id as i64
{
eprintln!(
"Specified transaction {} doesn't match current active transaction {}",
expected_transaction_id, current_transaction_id
);
std::process::exit(1);
}
let actual_transaction_id = current_transaction_id;
let response = client
.commit_transaction()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to commit transaction",
1,
))
.unwrap();
if !no_wait {
loop {
let stats = client
.get_pipeline_stats()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline stats",
1,
))
.unwrap();
let current_transaction_id = stats.into_inner().global_metrics.transaction_id;
if current_transaction_id != actual_transaction_id {
break;
}
sleep(Duration::from_secs(1)).await;
}
}
match format {
OutputFormat::Text => {
println!("Transaction committed successfully.");
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize commit response")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
PipelineAction::Diff {
name,
program_path,
stdin,
runtime_version,
} => {
let program_code = read_program_code(program_path, stdin)
.await
.unwrap_or_else(|()| std::process::exit(1));
let response = client
.post_pipeline_diff()
.pipeline_name(name.clone())
.body(PipelineDiffRequest {
program_code,
runtime_version,
})
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to compute pipeline diff",
1,
))
.unwrap();
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("failed to serialize pipeline diff")
);
}
PipelineAction::Rebalance { name } => {
client
.post_pipeline_rebalance()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to initiate rebalancing",
1,
))
.unwrap();
println!("Initiated rebalancing for pipeline {name}.");
}
PipelineAction::ClockAdvance { name, delta_ms } => {
let response = client
.clock_advance()
.pipeline_name(name.clone())
.body(ClockAdvanceRequest { delta_ms })
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to advance clock",
1,
))
.unwrap();
let body = response.into_inner();
match format {
OutputFormat::Text => {
println!("NOW() = {} ({} ms)", body.now, body.now_ms);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&body)
.expect("failed to serialize clock advance response")
);
}
_ => {
eprintln!(
"Unsupported output format, falling back to text: {}",
format
);
println!("NOW() = {} ({} ms)", body.now, body.now_ms);
std::process::exit(1);
}
}
}
PipelineAction::StartCompaction { name } => {
client
.post_pipeline_start_compaction()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to initiate compaction",
1,
))
.unwrap();
println!("Initiated compaction for pipeline {name}.");
}
PipelineAction::Bench { args } => bench::bench(client, format, args).await,
PipelineAction::DismissError { name } => {
let response = client
.post_pipeline_dismiss_error()
.pipeline_name(name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to dismiss pipeline deployment error",
1,
))
.unwrap();
println!("Pipeline deployment error dismissed successfully.");
trace!("{:#?}", response);
}
PipelineAction::Events { name, selector } => {
let response = client
.list_pipeline_events()
.pipeline_name(name.clone())
.selector(selector)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Unable to retrieve pipeline events",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
if selector == PipelineMonitorEventFieldSelector::All {
let mut rows = vec![];
rows.push([
"event_id".to_string(),
"recorded_at".to_string(),
"deployment_resources_status".to_string(),
"deployment_resources_status_details".to_string(),
"deployment_resources_desired_status".to_string(),
"deployment_runtime_status".to_string(),
"deployment_runtime_status_details".to_string(),
"deployment_runtime_desired_status".to_string(),
"deployment_has_error".to_string(),
"deployment_error".to_string(),
"program_status".to_string(),
"storage_status".to_string(),
"storage_status_details".to_string(),
]);
for event in response.iter() {
rows.push([
event.event_id.to_string(),
format!(
"{} ({:.0}s ago)",
event.recorded_at,
(Utc::now() - event.recorded_at).as_seconds_f64()
),
event.deployment_resources_status.to_string(),
event
.deployment_resources_status_details
.as_ref()
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
event.deployment_resources_desired_status.to_string(),
event
.deployment_runtime_status
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
event
.deployment_runtime_status_details
.as_ref()
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
event
.deployment_runtime_desired_status
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
event.deployment_has_error.to_string(),
event
.deployment_error
.as_ref()
.map(|v| format!("{v:?}"))
.unwrap_or("(none)".to_string()),
event.program_status.to_string(),
event.storage_status.to_string(),
event
.storage_status_details
.as_ref()
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
} else {
let mut rows = vec![];
rows.push([
"event_id".to_string(),
"recorded_at".to_string(),
"deployment_resources_status".to_string(),
"deployment_resources_desired_status".to_string(),
"deployment_runtime_status".to_string(),
"deployment_runtime_desired_status".to_string(),
"deployment_has_error".to_string(),
"program_status".to_string(),
"storage_status".to_string(),
]);
for event in response.iter() {
rows.push([
event.event_id.to_string(),
format!(
"{} ({:.0}s ago)",
event.recorded_at,
(Utc::now() - event.recorded_at).as_seconds_f64()
),
event.deployment_resources_status.to_string(),
event.deployment_resources_desired_status.to_string(),
event
.deployment_runtime_status
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
event
.deployment_runtime_desired_status
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
event.deployment_has_error.to_string(),
event.program_status.to_string(),
event.storage_status.to_string(),
]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline events")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
PipelineAction::Event {
name,
event_id,
selector,
} => {
let response = client
.get_pipeline_event()
.pipeline_name(name.clone())
.event_id(event_id)
.selector(selector)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Unable to retrieve pipeline event",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let mut rows = vec![];
rows.push(["Field".to_string(), "Value".to_string()]);
rows.push(["event_id".to_string(), response.event_id.to_string()]);
rows.push([
"recorded_at".to_string(),
format!(
"{} ({:.0}s ago)",
response.recorded_at,
(Utc::now() - response.recorded_at).as_seconds_f64()
),
]);
rows.push([
"deployment_resources_status".to_string(),
response.deployment_resources_status.to_string(),
]);
if selector == PipelineMonitorEventFieldSelector::All {
rows.push([
"deployment_resources_status_details".to_string(),
response
.deployment_resources_status_details
.as_ref()
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
]);
}
rows.push([
"deployment_resources_desired_status".to_string(),
response.deployment_resources_desired_status.to_string(),
]);
rows.push([
"deployment_runtime_status".to_string(),
response
.deployment_runtime_status
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
]);
if selector == PipelineMonitorEventFieldSelector::All {
rows.push([
"deployment_runtime_status_details".to_string(),
response
.deployment_runtime_status_details
.as_ref()
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
]);
}
rows.push([
"deployment_runtime_desired_status".to_string(),
response
.deployment_runtime_desired_status
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
]);
rows.push([
"deployment_has_error".to_string(),
response.deployment_has_error.to_string(),
]);
if selector == PipelineMonitorEventFieldSelector::All {
rows.push([
"deployment_error".to_string(),
response
.deployment_error
.as_ref()
.map(|v| format!("{v:?}"))
.unwrap_or("(none)".to_string()),
]);
}
rows.push([
"program_status".to_string(),
response.program_status.to_string(),
]);
rows.push([
"storage_status".to_string(),
response.storage_status.to_string(),
]);
if selector == PipelineMonitorEventFieldSelector::All {
rows.push([
"storage_status_details".to_string(),
response
.storage_status_details
.as_ref()
.map(|v| v.to_string())
.unwrap_or("(none)".to_string()),
]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline events")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
}
}
async fn connector_action(
client: &Client,
pipeline_name: String,
relation: &str,
connector: &str,
relation_is_table: bool,
action: &'static str,
past: &'static str,
) {
if relation_is_table {
client
.post_pipeline_input_connector_action()
.pipeline_name(pipeline_name)
.table_name(relation)
.connector_name(connector)
.action(action)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to start or pause table connector",
1,
))
.unwrap();
println!("Table {relation} connector {connector} {past} successfully.");
} else {
client
.post_pipeline_output_connector_action()
.pipeline_name(pipeline_name)
.view_name(relation)
.connector_name(connector)
.action(action)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to start or pause view connector",
1,
))
.unwrap();
println!("View {relation} connector {connector} {past} successfully.");
}
}
async fn connector(
format: OutputFormat,
pipeline_name: String,
relation: &str,
connector: &str,
action: ConnectorAction,
client: Client,
) {
let rc = client
.get_pipeline()
.pipeline_name(pipeline_name.clone())
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get pipeline config",
1,
))
.map(|response| response.program_info.clone())
.unwrap()
.unwrap();
let full_connector_name = format!("{relation}.{connector}");
let relation_is_table = rc
.input_connectors
.iter()
.any(|(name, _c)| *name == full_connector_name);
let relation_is_view = rc
.output_connectors
.iter()
.any(|(name, _c)| *name == full_connector_name);
if !relation_is_table && !relation_is_view {
eprintln!(
"No connector named {connector} found for relation {relation} in pipeline {pipeline_name}"
);
std::process::exit(1);
}
match action {
ConnectorAction::Start => {
connector_action(
&client,
pipeline_name,
relation,
connector,
relation_is_table,
"start",
"started",
)
.await;
}
ConnectorAction::Pause => {
connector_action(
&client,
pipeline_name,
relation,
connector,
relation_is_table,
"pause",
"paused",
)
.await;
}
ConnectorAction::Stats if relation_is_table => {
let response = client
.get_pipeline_input_connector_status()
.pipeline_name(pipeline_name)
.table_name(relation)
.connector_name(connector)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get table connector stats",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let stats_value = serde_json::to_value(response.as_ref())
.expect("Failed to serialize input connector stats");
let table = json_to_table(&stats_value)
.collapse()
.into_pool_table()
.to_string();
println!("{}", table);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(response.as_ref())
.expect("Failed to serialize input connector stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
ConnectorAction::Stats => {
let response = client
.get_pipeline_output_connector_status()
.pipeline_name(pipeline_name)
.view_name(relation)
.connector_name(connector)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get view connector stats",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let stats_value = serde_json::to_value(response.as_ref())
.expect("Failed to serialize output connector stats");
let table = json_to_table(&stats_value)
.collapse()
.into_pool_table()
.to_string();
println!("{}", table);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(response.as_ref())
.expect("Failed to serialize output connector stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
};
}
fn format_program_errors(program_error: &ProgramError) -> String {
let mut output = String::new();
let mut has_output = false;
if let Some(sql) = &program_error.sql_compilation {
has_output = true;
output.push_str(&format!("SQL compiler exit code: {}\n", sql.exit_code));
if sql.messages.is_empty() {
output.push_str("No SQL compiler messages.\n");
} else {
for message in &sql.messages {
let severity = if message.warning { "warning" } else { "error" };
let error_type = &message.error_type;
let message_text = &message.message;
let start_line = message.start_line_number;
let start_column = message.start_column;
writeln!(
&mut output,
"{severity}: {error_type}: {message_text} (line {start_line}, column {start_column})",
)
.expect("writing to String should not fail");
if let Some(snippet) = &message.snippet
&& !snippet.is_empty()
{
output.push_str(snippet);
if !snippet.ends_with('\n') {
output.push('\n');
}
}
}
}
}
if let Some(rust) = &program_error.rust_compilation {
if has_output {
output.push('\n');
}
has_output = true;
output.push_str(&format!("Rust compiler exit code: {}\n", rust.exit_code));
if !rust.stdout.is_empty() {
output.push_str("\nstdout:\n");
output.push_str(&rust.stdout);
if !rust.stdout.ends_with('\n') {
output.push('\n');
}
}
if !rust.stderr.is_empty() {
output.push_str("\nstderr:\n");
output.push_str(&rust.stderr);
if !rust.stderr.ends_with('\n') {
output.push('\n');
}
}
if rust.stdout.is_empty() && rust.stderr.is_empty() {
output.push_str("No Rust compiler stdout or stderr.\n");
}
}
if let Some(system_error) = &program_error.system_error {
if has_output {
output.push('\n');
}
has_output = true;
output.push_str("System error:\n");
output.push_str(system_error);
if !system_error.ends_with('\n') {
output.push('\n');
}
}
if !has_output {
output.push_str("No compilation errors or warnings.\n");
}
output
}
async fn program(format: OutputFormat, action: ProgramAction, client: Client) {
match action {
ProgramAction::Get {
name,
udf_rs,
udf_toml,
} => {
let response = client
.get_pipeline()
.pipeline_name(name)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get program",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
if !udf_rs && !udf_toml {
println!("{}", response.program_code.clone().unwrap());
}
if udf_rs {
println!("{}", response.udf_rust.clone().unwrap());
}
if udf_toml {
println!("{}", response.udf_toml.clone().unwrap());
}
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
ProgramAction::Config { name } => {
let response = client
.get_pipeline()
.pipeline_name(name)
.selector(PipelineFieldSelector::Status)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get program config",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let value = serde_json::to_value(&response.program_config).unwrap();
let table = json_to_table(&value).collapse().to_string();
println!("{}", table);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.program_config)
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
ProgramAction::SetConfig {
name,
profile,
runtime_version,
use_platform_compiler,
} => {
let pp = PatchPipeline {
description: None,
tags: None,
name: None,
program_code: None,
udf_rust: None,
udf_toml: None,
program_config: Some(ProgramConfig {
profile,
cache: true,
runtime_version,
use_platform_compiler,
}),
runtime_config: None,
deployment_config: None,
};
let response = client
.patch_pipeline()
.pipeline_name(name)
.body(pp)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to set program config",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!("Program config updated successfully.");
let value = serde_json::to_value(&response.program_config).unwrap();
let table = json_to_table(&value).collapse().to_string();
println!("{}", table);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.program_config)
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
ProgramAction::Status { name } => {
let response = client
.get_pipeline()
.pipeline_name(name)
.selector(PipelineFieldSelector::Status)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get program status",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!(
"{:#?} (version {}, last updated {})",
response.program_status,
response.program_version.0,
response.program_status_since
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
ProgramAction::Errors { name } => {
let response = client
.get_pipeline()
.pipeline_name(name)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to get program compilation errors",
1,
))
.unwrap();
let Some(program_error) = response.into_inner().program_error else {
eprintln!(
"Pipeline response did not include program errors. {}",
UPGRADE_NOTICE
);
std::process::exit(1);
};
match format {
OutputFormat::Text => {
print!("{}", format_program_errors(&program_error));
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&program_error)
.expect("Failed to serialize program errors")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
ProgramAction::Set {
name,
program_path,
udf_rs,
udf_toml,
stdin,
} => {
if let (Ok(program_code), Ok(udf_rust), Ok(udf_toml)) = (
read_program_code(program_path, stdin).await,
read_file(udf_rs).await,
read_file(udf_toml).await,
) {
let pp = PatchPipeline {
description: None,
tags: None,
name: None,
program_code,
udf_rust,
udf_toml,
program_config: None,
runtime_config: None,
deployment_config: None,
};
let response = client
.patch_pipeline()
.pipeline_name(name)
.body(pp)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Failed to set program code",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
println!("Program updated successfully.");
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize pipeline stats")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
} else {
std::process::exit(1);
}
}
}
}
async fn cluster(format: OutputFormat, action: ClusterAction, client: Client) {
match action {
ClusterAction::Events => {
let response = client
.list_cluster_events()
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Unable to retrieve cluster events",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let mut rows = vec![];
rows.push([
"id".to_string(),
"recorded_at".to_string(),
"all_healthy".to_string(),
"api_status".to_string(),
"compiler_status".to_string(),
"runner_status".to_string(),
]);
for event in response.iter() {
rows.push([
event.id.to_string(),
format!(
"{} ({:.0}s ago)",
event.recorded_at,
(Utc::now() - event.recorded_at).as_seconds_f64()
),
event.all_healthy.to_string(),
event.api_status.to_string(),
event.compiler_status.to_string(),
event.runner_status.to_string(),
]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize cluster events")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
ClusterAction::Event { id, selector } => {
let response = client
.get_cluster_event()
.event_id(id)
.selector(selector)
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Unable to retrieve cluster event",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let mut rows = vec![];
rows.push(["Field".to_string(), "Value".to_string()]);
rows.push(["id".to_string(), response.id.to_string()]);
rows.push([
"recorded_at".to_string(),
format!(
"{} ({:.0}s ago)",
response.recorded_at,
(Utc::now() - response.recorded_at).as_seconds_f64()
),
]);
if let Some(stale) = response.stale {
rows.push(["stale".to_string(), stale.to_string()]);
}
rows.push(["all_healthy".to_string(), response.all_healthy.to_string()]);
rows.push(["api_status".to_string(), response.api_status.to_string()]);
if let Some(value) = &response.api_self_info {
rows.push(["api_self_info".to_string(), value.to_string()]);
}
if let Some(value) = &response.api_resources_info {
rows.push(["api_resources_info".to_string(), value.to_string()]);
}
rows.push([
"compiler_status".to_string(),
response.compiler_status.to_string(),
]);
if let Some(value) = &response.compiler_self_info {
rows.push(["compiler_self_info".to_string(), value.to_string()]);
}
if let Some(value) = &response.compiler_resources_info {
rows.push(["compiler_resources_info".to_string(), value.to_string()]);
}
rows.push([
"runner_status".to_string(),
response.runner_status.to_string(),
]);
if let Some(value) = &response.runner_self_info {
rows.push(["runner_self_info".to_string(), value.to_string()]);
}
if let Some(value) = &response.runner_resources_info {
rows.push(["runner_resources_info".to_string(), value.to_string()]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize cluster events")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
ClusterAction::Health => {
let health = match client.get_cluster_health().send().await {
Ok(response) => response.into_inner(),
Err(Error::ErrorResponse(response)) => response.into_inner(),
Err(e) => {
eprintln!(
"Unable to retrieve cluster health from {}: {e}",
client.baseurl()
);
std::process::exit(1);
}
};
match format {
OutputFormat::Text => {
let now = Utc::now();
if health.stale {
eprintln!(
"Warning: no cluster monitor event for over {} minutes, so the \
statuses below are the last recorded ones, not current ones. On \
Feldera Enterprise, the cluster monitor runs inside the Kubernetes \
runner, which may be down.",
health.stale_after_seconds / 60
);
}
let mut rows = vec![];
rows.push([
"service".to_string(),
"healthy".to_string(),
"unchanged_since".to_string(),
"checked_at".to_string(),
"message".to_string(),
]);
for (service, status) in [
("api", &health.api),
("compiler", &health.compiler),
("runner", &health.runner),
] {
rows.push([
service.to_string(),
status.healthy.to_string(),
format!(
"{} ({:.0}s ago)",
status.unchanged_since,
(now - status.unchanged_since).as_seconds_f64()
),
format!(
"{} ({:.0}s ago)",
status.checked_at,
(now - status.checked_at).as_seconds_f64()
),
status.message.clone(),
]);
}
println!(
"{}",
Builder::from_iter(rows).build().with(Style::rounded())
);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&health)
.expect("Failed to serialize cluster health")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
if !health.all_healthy {
std::process::exit(2);
}
}
ClusterAction::Config => {
let response = client
.get_config()
.send()
.await
.map_err(handle_errors_fatal(
client.baseurl().to_string(),
"Unable to retrieve platform configuration",
1,
))
.unwrap();
match format {
OutputFormat::Text => {
let value = json!({
"edition": response.edition,
"version": response.version,
"revision": response.revision,
"runtime_revision": response.runtime_revision,
"build_source": response.build_source,
"build_timestamp": response.build_info.build_timestamp,
"rustc_version": response.build_info.rustc_version,
"cargo_target_triple": response.build_info.cargo_target_triple,
"license_validity": response.license_validity,
"update_info": response.update_info,
"unstable_features": response.unstable_features,
"changelog_url": response.changelog_url,
});
let table = json_to_table(&value).collapse().to_string();
println!("{}", table);
}
OutputFormat::Json => {
println!(
"{}",
serde_json::to_string_pretty(&response.into_inner())
.expect("Failed to serialize platform configuration")
);
}
_ => {
eprintln!("Unsupported output format: {}", format);
std::process::exit(1);
}
}
}
}
}
fn install_crypto_provider() {
let _ = rustls::crypto::CryptoProvider::install_default(
rustls::crypto::aws_lc_rs::default_provider(),
);
}
fn main() {
install_crypto_provider();
init_logging("warn");
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
CompleteEnv::with_factory(Cli::command).complete();
let mut cli = Cli::parse();
if cli.host.ends_with("/") {
cli.host = cli.host.trim_end_matches('/').to_string();
}
if cli.auth.is_none()
&& cli.oidc_token_file.is_none()
&& std::env::var_os("FELDERA_AUTH_TOKEN_COMMAND").is_some()
{
warn!(
"FELDERA_AUTH_TOKEN_COMMAND is no longer read. Point FELDERA_OIDC_TOKEN_FILE at a file holding the token, or pass the command's output as --auth \"$(...)\"."
);
}
let client_opts = ClientOpts::from_cli(&cli);
let client = move || {
make_client(client_opts)
.map_err(|e| {
eprintln!("Failed to create HTTP client: {}", e);
std::process::exit(1);
})
.unwrap()
};
match cli.command {
Commands::Apikey { action } => api_key_commands(cli.format, action, client()).await,
Commands::OidcTrust { action } => {
oidc_trust_commands(cli.format, action, client()).await
}
Commands::Tenant { action } => tenant_commands(cli.format, action, client()).await,
Commands::Member { action } => member_commands(cli.format, action, client()).await,
Commands::Pipelines => pipelines(cli.format, client()).await,
Commands::Pipeline(action) => pipeline(cli.format, action, client()).await,
Commands::ValidateProgram {
program_path,
stdin,
runtime_version,
ir,
} => validate_program(program_path, stdin, runtime_version, ir, client()).await,
Commands::Cluster { action } => cluster(cli.format, action, client()).await,
Commands::Debug { action } => debug::debug(cli.format, action, client()).await,
}
})
}
fn init_logging(default_level: &str) {
let filter =
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default_level));
let _ = LogTracer::init();
let _ = tracing_subscriber::registry()
.with(filter)
.with(
tracing_subscriber::fmt::layer()
.with_target(false)
.without_time()
.with_writer(std::io::stderr),
)
.try_init();
}
#[cfg(test)]
mod tests {
use super::{
Client, ClientInfo, ClientOpts, RuntimeConfig, RuntimeConfigKey, format_program_errors,
install_crypto_provider, make_client, patch_runtime_config, read_oidc_token_file,
};
use feldera_rest_api::types::{
ProgramError, RustCompilationInfo, SqlCompilationInfo, SqlCompilerMessage,
};
use feldera_types::duration::ConfigDuration;
use feldera_types::size::ConfigSize;
use std::io::Write;
use std::time::Duration;
#[test]
fn setting_a_duration_key_stores_a_duration() {
let mut config = RuntimeConfig::default();
patch_runtime_config(&mut config, RuntimeConfigKey::MaxBufferingDelay, "10ms").unwrap();
patch_runtime_config(&mut config, RuntimeConfigKey::ClockResolution, "2s").unwrap();
assert_eq!(
config.max_buffering_delay,
Some(ConfigDuration::from_millis(10))
);
assert_eq!(config.clock_resolution, Some(ConfigDuration::from_secs(2)));
}
#[test]
fn setting_a_size_key_stores_a_size() {
let mut config = RuntimeConfig::default();
patch_runtime_config(&mut config, RuntimeConfigKey::MemoryMax, "16Gi").unwrap();
patch_runtime_config(&mut config, RuntimeConfigKey::MemoryMbMin, "4000").unwrap();
patch_runtime_config(&mut config, RuntimeConfigKey::StorageMax, "400G").unwrap();
patch_runtime_config(&mut config, RuntimeConfigKey::MaxRss, "12G").unwrap();
patch_runtime_config(&mut config, RuntimeConfigKey::Storage, "true").unwrap();
patch_runtime_config(&mut config, RuntimeConfigKey::MinSpillSize, "10Mi").unwrap();
assert_eq!(config.resources.memory_max, Some(ConfigSize::from_gib(16)));
assert_eq!(config.resources.memory_min, Some(ConfigSize::from_mb(4000)));
assert_eq!(
config.resources.storage_max,
Some(ConfigSize::from_mb(400_000))
);
assert_eq!(config.max_rss, Some(ConfigSize::from_mb(12_000)));
assert_eq!(
config.storage.as_ref().unwrap().min_spill_size,
Some(ConfigSize::from_mib(10))
);
let error =
patch_runtime_config(&mut config, RuntimeConfigKey::MemoryMax, "16 GB").unwrap_err();
assert!(
error.contains("memory_max") && error.contains("unknown unit"),
"{error}"
);
let error =
patch_runtime_config(&mut config, RuntimeConfigKey::MemoryMax, "16000").unwrap_err();
assert!(error.contains("`16000M`"), "{error}");
patch_runtime_config(&mut config, RuntimeConfigKey::MinSpillSize, "4096").unwrap();
assert_eq!(
config.storage.as_ref().unwrap().min_spill_size,
Some(ConfigSize::from_bytes(4096))
);
}
#[test]
fn checkpoint_interval_can_disable_checkpointing() {
let mut config = RuntimeConfig::default();
patch_runtime_config(&mut config, RuntimeConfigKey::CheckpointInterval, "30s").unwrap();
assert_eq!(
config.fault_tolerance.checkpoint_interval,
Some(Some(ConfigDuration::from_secs(30)))
);
for disable in ["none", "null"] {
patch_runtime_config(&mut config, RuntimeConfigKey::CheckpointInterval, disable)
.unwrap();
assert_eq!(
config.fault_tolerance.checkpoint_interval,
Some(None),
"`{disable}` should disable periodic checkpoints"
);
}
}
#[test]
fn checkpoint_interval_still_accepts_a_bare_number_of_seconds() {
let mut config = RuntimeConfig::default();
patch_runtime_config(&mut config, RuntimeConfigKey::CheckpointInterval, "90").unwrap();
assert_eq!(
config.fault_tolerance.checkpoint_interval,
Some(Some(ConfigDuration::from_secs(90)))
);
patch_runtime_config(&mut config, RuntimeConfigKey::CheckpointInterval, "0").unwrap();
assert_eq!(config.fault_tolerance.checkpoint_interval, Some(None));
patch_runtime_config(&mut config, RuntimeConfigKey::CheckpointInterval, "0s").unwrap();
assert_eq!(
config.fault_tolerance.checkpoint_interval,
Some(Some(ConfigDuration::ZERO))
);
}
#[test]
fn a_rejected_value_explains_itself() {
let mut config = RuntimeConfig::default();
let error =
patch_runtime_config(&mut config, RuntimeConfigKey::MaxBufferingDelay, "10 sec")
.unwrap_err();
assert!(
error.contains("max_buffering_delay"),
"should name the key as typed, got: {error}"
);
assert!(
error.contains("10 sec"),
"should quote the offending value, got: {error}"
);
assert!(
error.contains("unit"),
"should say a unit was the problem, got: {error}"
);
let error =
patch_runtime_config(&mut config, RuntimeConfigKey::Workers, "many").unwrap_err();
assert!(
error.contains("workers") && error.contains("many"),
"should name the key and the value, got: {error}"
);
}
const VALID_TEST_PEM: &str = "-----BEGIN CERTIFICATE-----\n\
MIIBhTCCASugAwIBAgIUSkl1zF8JmnzQL4R2lK2RgqxWVCQwCgYIKoZIzj0EAwIw\n\
FDESMBAGA1UEAwwJVGVzdCBSb290MB4XDTI0MDEwMTAwMDAwMFoXDTM0MDEwMTAw\n\
MDAwMFowFDESMBAGA1UEAwwJVGVzdCBSb290MFkwEwYHKoZIzj0CAQYIKoZIzj0D\n\
AQcDQgAEu/n2TjZ+9/IYeiI4lL9zFqpTtq+gg3i5g7ZGY7h7Z7N9wz7T2mEbq5u9\n\
6Lw1D5i9a7vhB9oXy5AE0AVGekt+cqNjMGEwHQYDVR0OBBYEFHZkI7WQpjR9X5+y\n\
M0l7m5s8B1bkMB8GA1UdIwQYMBaAFHZkI7WQpjR9X5+yM0l7m5s8B1bkMA8GA1Ud\n\
EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMCA0gAMEUCIQDN\n\
2cA4cQKG/9XG9B5mJdQ3L+sVu+8fF7kS5q3cq1pPMQIgBmV7f7ZQ6kqoK0qk0Cg9\n\
aC3Oy4iVrYGOq9v6uP9iblE=\n\
-----END CERTIFICATE-----\n";
fn tls_test_opts(tls_cert: std::path::PathBuf) -> ClientOpts {
ClientOpts {
host: "https://example.invalid".to_string(),
insecure: false,
tls_cert: Some(tls_cert),
auth: None,
oidc_token_file: None,
headers: Vec::new(),
timeout_secs: None,
tenant: None,
retries: 0,
}
}
#[test]
fn make_client_tls_cert_missing_file_returns_error() {
let missing = std::path::PathBuf::from("/definitely/not/a/real/path-fda-test.pem");
let err = make_client(tls_test_opts(missing))
.expect_err("non-existent cert path must produce an error");
let msg = err.to_string();
assert!(
msg.contains("Failed to read TLS certificate file"),
"unexpected error: {msg}"
);
}
#[test]
fn make_client_tls_cert_invalid_pem_returns_error() {
let mut file = tempfile::NamedTempFile::new().expect("create temp file");
file.write_all(b"this is not a PEM certificate")
.expect("write temp file");
let err = make_client(tls_test_opts(file.path().to_path_buf()))
.expect_err("garbage cert contents must produce an error");
let msg = err.to_string();
assert!(
msg.contains("Failed to parse TLS certificate file")
|| msg.contains("did not contain any PEM-encoded certificates"),
"unexpected error: {msg}"
);
}
#[test]
fn make_client_tls_cert_valid_pem_builds_client() {
let mut file = tempfile::NamedTempFile::new().expect("create temp file");
file.write_all(VALID_TEST_PEM.as_bytes())
.expect("write temp file");
let result = make_client(tls_test_opts(file.path().to_path_buf()));
if let Err(err) = &result {
let msg = err.to_string();
assert!(
!msg.contains("Failed to read TLS certificate file"),
"unexpected error from valid PEM path: {msg}"
);
assert!(
!msg.contains("did not contain any PEM-encoded certificates"),
"unexpected empty-bundle error from valid PEM path: {msg}"
);
}
}
#[test]
fn oidc_token_file_trims_contents() {
let mut file = tempfile::NamedTempFile::new().expect("create temp file");
file.write_all(b" tok-123\n").expect("write temp file");
let token = read_oidc_token_file(file.path()).expect("readable file");
assert_eq!(token, "tok-123");
}
#[test]
fn oidc_token_file_whitespace_only_is_error() {
let mut file = tempfile::NamedTempFile::new().expect("create temp file");
file.write_all(b"\n\t \n").expect("write temp file");
let err = read_oidc_token_file(file.path()).expect_err("blank file must error");
assert!(err.to_string().contains("is empty"), "{err}");
}
#[test]
fn oidc_token_file_missing_is_error() {
let missing = std::path::Path::new("/definitely/not/a/real/path-fda-token");
let err = read_oidc_token_file(missing).expect_err("missing file must error");
let msg = err.to_string();
assert!(msg.contains("failed to read OIDC token file"), "{msg}");
assert!(
msg.contains("path-fda-token"),
"path should be named: {msg}"
);
}
#[test]
fn oidc_token_file_second_line_is_error() {
let mut file = tempfile::NamedTempFile::new().expect("create temp file");
file.write_all(b"tok-1\ntok-2\n").expect("write temp file");
let err = read_oidc_token_file(file.path()).expect_err("two lines must error");
let msg = err.to_string();
assert!(msg.contains("does not hold a single-line token"), "{msg}");
assert!(msg.contains("a line break at byte 5"), "{msg}");
}
fn token_file_opts(host: &str, oidc_token_file: std::path::PathBuf) -> ClientOpts {
ClientOpts {
host: host.to_string(),
insecure: false,
tls_cert: None,
auth: None,
oidc_token_file: Some(oidc_token_file),
headers: Vec::new(),
timeout_secs: None,
tenant: None,
retries: 0,
}
}
#[test]
fn make_client_reads_token_file_for_https_only() {
install_crypto_provider();
let missing = std::path::PathBuf::from("/definitely/not/a/real/path-fda-token");
make_client(token_file_opts("http://example.invalid", missing.clone()))
.expect("http host must not read the token file");
let err = make_client(token_file_opts("https://example.invalid", missing))
.expect_err("https host must fail on a missing token file");
assert!(
err.to_string().contains("failed to read OIDC token file"),
"{err}"
);
}
#[test]
fn make_client_rejects_blank_token_file_for_https() {
install_crypto_provider();
let file = tempfile::NamedTempFile::new().expect("create temp file");
let err = make_client(token_file_opts(
"https://example.invalid",
file.path().to_path_buf(),
))
.expect_err("blank token file must fail rather than send nothing");
assert!(err.to_string().contains("is empty"), "{err}");
}
#[test]
fn format_program_errors_empty() {
let output = format_program_errors(&ProgramError {
sql_compilation: None,
rust_compilation: None,
system_error: None,
});
assert_eq!(output, "No compilation errors or warnings.\n");
}
#[test]
fn format_program_errors_sql_messages() {
let output = format_program_errors(&ProgramError {
sql_compilation: Some(SqlCompilationInfo {
exit_code: 1,
messages: vec![SqlCompilerMessage {
start_line_number: 2,
start_column: 4,
end_line_number: 2,
end_column: 8,
warning: true,
error_type: "PRIMARY KEY cannot be nullable".to_string(),
message: "PRIMARY KEY column 'C' has type INTEGER, which is nullable"
.to_string(),
snippet: Some(" 2| c INT PRIMARY KEY\n ^^^^^\n".to_string()),
}],
}),
rust_compilation: None,
system_error: None,
});
assert_eq!(
output,
concat!(
"SQL compiler exit code: 1\n",
"warning: PRIMARY KEY cannot be nullable: PRIMARY KEY column 'C' has type INTEGER, which is nullable (line 2, column 4)\n",
" 2| c INT PRIMARY KEY\n",
" ^^^^^\n",
)
);
}
#[test]
fn format_program_errors_rust_output() {
let output = format_program_errors(&ProgramError {
sql_compilation: None,
rust_compilation: Some(RustCompilationInfo {
exit_code: 101,
messages: Vec::new(),
stdout: "checking pipeline\n".to_string(),
stderr: "error: failed to compile".to_string(),
}),
system_error: None,
});
assert_eq!(
output,
"Rust compiler exit code: 101\n\
\n\
stdout:\n\
checking pipeline\n\
\n\
stderr:\n\
error: failed to compile\n"
);
}
#[test]
fn format_program_errors_system_error() {
let output = format_program_errors(&ProgramError {
sql_compilation: None,
rust_compilation: None,
system_error: Some("compiler service unavailable".to_string()),
});
assert_eq!(output, "System error:\ncompiler service unavailable\n");
}
fn spawn_recording_server() -> (u16, std::sync::mpsc::Receiver<String>) {
use std::io::BufRead;
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind test server");
let port = listener.local_addr().expect("test server address").port();
let (record, recorded) = std::sync::mpsc::channel();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut stream) = stream else { continue };
let mut reader =
std::io::BufReader::new(stream.try_clone().expect("clone test connection"));
let mut head = String::new();
let mut line = String::new();
while matches!(reader.read_line(&mut line), Ok(bytes) if bytes > 2) {
head.push_str(&line);
line.clear();
}
if record.send(head).is_err() {
break;
}
let _ = stream.write_all(
b"HTTP/1.1 200 OK\r\n\
Content-Type: application/json\r\n\
Content-Length: 2\r\n\
Connection: close\r\n\r\n[]",
);
let _ = stream.flush();
}
});
(port, recorded)
}
fn local_test_opts(port: u16, headers: &[&str]) -> ClientOpts {
ClientOpts {
host: format!("http://127.0.0.1:{port}"),
insecure: false,
tls_cert: None,
auth: None,
oidc_token_file: None,
headers: headers.iter().map(|header| header.to_string()).collect(),
timeout_secs: Some(10),
tenant: None,
retries: 0,
}
}
fn build_test_client(opts: ClientOpts) -> Client {
install_crypto_provider();
make_client(opts).expect("build client")
}
async fn request_head(client: &Client, recorded: &std::sync::mpsc::Receiver<String>) -> String {
client
.list_pipelines()
.send()
.await
.expect("the test server answers with an empty list");
recorded
.recv_timeout(Duration::from_secs(10))
.expect("the server recorded the request")
.to_lowercase()
}
#[tokio::test]
async fn custom_headers_are_sent_with_every_request() {
let (port, recorded) = spawn_recording_server();
let client = build_test_client(local_test_opts(
port,
&["Cookie: AWSELBAuthSessionCookie-0=abc", "X-Trace-Id: 42"],
));
let head = request_head(&client, &recorded).await;
assert!(
head.contains("cookie: awselbauthsessioncookie-0=abc"),
"{head}"
);
assert!(head.contains("x-trace-id: 42"), "{head}");
}
#[tokio::test]
async fn custom_header_replaces_the_clients_own() {
let (port, recorded) = spawn_recording_server();
let mut opts = local_test_opts(port, &["Feldera-Tenant: chosen-by-header"]);
opts.tenant = Some("chosen-by-flag".to_string());
let client = build_test_client(opts);
let head = request_head(&client, &recorded).await;
assert_eq!(
head.matches("feldera-tenant:").count(),
1,
"the tenant must be named once: {head}"
);
assert!(head.contains("feldera-tenant: chosen-by-header"), "{head}");
}
#[tokio::test]
async fn repeating_a_header_name_keeps_the_later_value() {
let (port, recorded) = spawn_recording_server();
let client = build_test_client(local_test_opts(
port,
&["X-Forwarded-For: 10.0.0.1", "X-Forwarded-For: 10.0.0.2"],
));
let head = request_head(&client, &recorded).await;
assert_eq!(
head.matches("x-forwarded-for:").count(),
1,
"the header must be sent once: {head}"
);
assert!(head.contains("x-forwarded-for: 10.0.0.2"), "{head}");
}
#[tokio::test]
async fn custom_headers_reach_the_websocket_upgrade() {
use reqwest_websocket::RequestBuilderExt;
let (port, recorded) = spawn_recording_server();
let client = build_test_client(local_test_opts(port, &["Cookie: session=abc"]));
let _ = client
.client()
.get(format!("http://127.0.0.1:{port}/v0/pipelines/p/query"))
.upgrade()
.send()
.await;
let head = recorded
.recv_timeout(Duration::from_secs(10))
.expect("the server recorded the upgrade request")
.to_lowercase();
assert!(head.contains("upgrade: websocket"), "{head}");
assert!(head.contains("cookie: session=abc"), "{head}");
}
#[tokio::test]
async fn credential_header_travels_where_auth_is_withheld() {
let (port, recorded) = spawn_recording_server();
let mut opts = local_test_opts(port, &["Authorization: Bearer from-header"]);
opts.auth = Some("apikey:withheld".to_string());
let client = build_test_client(opts);
let head = request_head(&client, &recorded).await;
assert!(head.contains("authorization: bearer from-header"), "{head}");
assert!(
!head.contains("apikey:withheld"),
"`--auth` must stay off an unencrypted connection: {head}"
);
}
#[test]
fn unparsable_header_argument_fails_the_client() {
let err = make_client(local_test_opts(0, &["not-a-header"]))
.expect_err("a header without a colon must be rejected");
let msg = err.to_string();
assert!(msg.contains("not-a-header"), "{msg}");
assert!(msg.contains("Name: Value"), "{msg}");
}
}