diff --git a/crates/hyperqueue/src/client/commands/worker.rs b/crates/hyperqueue/src/client/commands/worker.rs index c1d890e85..c203417bf 100644 --- a/crates/hyperqueue/src/client/commands/worker.rs +++ b/crates/hyperqueue/src/client/commands/worker.rs @@ -281,24 +281,6 @@ pub struct WorkerStartOpts { /// It should *NOT* be placed on a network filesystem. #[arg(long)] pub work_dir: Option, - - /// The maximal parallel downloads for data objects - #[arg(long, default_value = "4")] - pub max_parallel_downloads: u32, - - /// The maximal data object download tries - /// - /// Specifies how many times the worker tries to download a data object - /// from the remote side before download is considered as failed. - #[arg(long, default_value = "8")] - pub max_download_tries: u32, - - #[arg(long, - default_value = "1s", value_parser = parse_hms_or_human_time, - help = duration_doc!("The delay between download attempts\n\nSets how long to wait between failed downloads of data object. This time is multiplied by the number of previous retries. Therefore between 4th and 5th retry it waits 4 * the given duration"), - value_name = "TIME") - ] - pub wait_between_download_tries: Duration, } pub async fn start_hq_worker( @@ -344,9 +326,6 @@ fn gather_configuration(opts: WorkerStartOpts) -> anyhow::Result anyhow::Result = resource.into_iter().map(|x| x.into_parsed_arg()).collect(); let cpu_resources = resources.iter().find(|x| x.name == CPU_RESOURCE_NAME); @@ -480,9 +451,6 @@ fn gather_configuration(opts: WorkerStartOpts) -> anyhow::Result serde_json::Value { time_limit, on_server_lost, group, - max_parallel_downloads, - max_download_tries, - wait_between_download_tries, extra: _, }, started, @@ -579,9 +576,6 @@ fn format_worker_info(worker_info: WorkerInfo) -> serde_json::Value { "listen_address": listen_address, "resources": format_resource_descriptor(&resources), "on_server_lost": crate::common::format::server_lost_policy_to_str(&on_server_lost), - "max_parallel_downloads": max_parallel_downloads, - "max_download_tries": max_download_tries, - "wait_between_download_tries": format_duration(wait_between_download_tries), }), "allocation": manager_info.map(|info| json!({ "manager": FormattedManagerType(info.manager), diff --git a/crates/pyhq/src/cluster/worker.rs b/crates/pyhq/src/cluster/worker.rs index 8fdb1e480..8f2840eea 100644 --- a/crates/pyhq/src/cluster/worker.rs +++ b/crates/pyhq/src/cluster/worker.rs @@ -6,10 +6,7 @@ use hyperqueue::common::utils::network::get_hostname; use tokio::task::LocalSet; use hyperqueue::worker::bootstrap::{finalize_configuration, initialize_worker}; -use tako::internal::worker::configuration::{ - DEFAULT_MAX_DOWNLOAD_TRIES, DEFAULT_MAX_PARALLEL_DOWNLOADS, - DEFAULT_WAIT_BETWEEN_DOWNLOAD_TRIES, OverviewConfiguration, -}; +use tako::internal::worker::configuration::OverviewConfiguration; use tako::resources::{ CPU_RESOURCE_NAME, ResourceDescriptor, ResourceDescriptorItem, ResourceDescriptorKind, ResourceIndex, @@ -55,9 +52,6 @@ impl RunningWorker { idle_timeout: None, time_limit: None, on_server_lost: ServerLostPolicy::Stop, - max_parallel_downloads: DEFAULT_MAX_PARALLEL_DOWNLOADS, - max_download_tries: DEFAULT_MAX_DOWNLOAD_TRIES, - wait_between_download_tries: DEFAULT_WAIT_BETWEEN_DOWNLOAD_TRIES, extra: Default::default(), }; finalize_configuration(&mut configuration); diff --git a/crates/tako/src/internal/scheduler/query.rs b/crates/tako/src/internal/scheduler/query.rs index 1e49cadcb..e61ade9d6 100644 --- a/crates/tako/src/internal/scheduler/query.rs +++ b/crates/tako/src/internal/scheduler/query.rs @@ -58,9 +58,6 @@ pub(crate) fn compute_new_worker_query( overview_configuration: Default::default(), idle_timeout: None, on_server_lost: ServerLostPolicy::Stop, - max_parallel_downloads: 0, - max_download_tries: 0, - wait_between_download_tries: Default::default(), extra: Default::default(), }; let worker = Worker::new(worker_id, configuration, &resource_map, now); diff --git a/crates/tako/src/internal/tests/integration/utils/worker.rs b/crates/tako/src/internal/tests/integration/utils/worker.rs index 99c9e4a98..79c3688fa 100644 --- a/crates/tako/src/internal/tests/integration/utils/worker.rs +++ b/crates/tako/src/internal/tests/integration/utils/worker.rs @@ -7,10 +7,7 @@ use std::time::Duration; use crate::internal::common::error::DsError; use crate::internal::common::resources::ResourceDescriptor; -use crate::internal::worker::configuration::{ - DEFAULT_MAX_DOWNLOAD_TRIES, DEFAULT_MAX_PARALLEL_DOWNLOADS, - DEFAULT_WAIT_BETWEEN_DOWNLOAD_TRIES, OverviewConfiguration, -}; +use crate::internal::worker::configuration::OverviewConfiguration; use crate::launcher::{StopReason, TaskBuildContext, TaskResult}; use crate::program::ProgramDefinition; use crate::worker::WorkerConfiguration; @@ -81,9 +78,6 @@ pub fn create_worker_configuration( }, idle_timeout, on_server_lost: ServerLostPolicy::Stop, - max_parallel_downloads: DEFAULT_MAX_PARALLEL_DOWNLOADS, - max_download_tries: DEFAULT_MAX_DOWNLOAD_TRIES, - wait_between_download_tries: DEFAULT_WAIT_BETWEEN_DOWNLOAD_TRIES, time_limit: None, extra, }, diff --git a/crates/tako/src/internal/tests/test_reactor.rs b/crates/tako/src/internal/tests/test_reactor.rs index 972281f76..54d374694 100644 --- a/crates/tako/src/internal/tests/test_reactor.rs +++ b/crates/tako/src/internal/tests/test_reactor.rs @@ -15,10 +15,7 @@ use crate::internal::tests::utils::shared::{res_kind_groups, res_kind_sum}; use crate::internal::tests::utils::sorted_vec; use crate::internal::tests::utils::task::{TaskBuilder, task_running_msg}; use crate::internal::tests::utils::workflows::{submit_example_1, submit_example_3}; -use crate::internal::worker::configuration::{ - DEFAULT_MAX_DOWNLOAD_TRIES, DEFAULT_MAX_PARALLEL_DOWNLOADS, - DEFAULT_WAIT_BETWEEN_DOWNLOAD_TRIES, OverviewConfiguration, -}; +use crate::internal::worker::configuration::OverviewConfiguration; use crate::resources::{ResourceAmount, ResourceDescriptorItem, ResourceIdMap}; use crate::tests::utils::env::{TestComm, TestEnv}; use crate::tests::utils::worker::WorkerBuilder; @@ -48,9 +45,6 @@ fn test_worker_add() { idle_timeout: None, time_limit: None, on_server_lost: ServerLostPolicy::Stop, - max_parallel_downloads: DEFAULT_MAX_PARALLEL_DOWNLOADS, - max_download_tries: DEFAULT_MAX_DOWNLOAD_TRIES, - wait_between_download_tries: DEFAULT_WAIT_BETWEEN_DOWNLOAD_TRIES, extra: Default::default(), group: "default".to_string(), }; @@ -108,9 +102,6 @@ fn test_worker_add() { idle_timeout: None, time_limit: None, on_server_lost: ServerLostPolicy::Stop, - max_parallel_downloads: DEFAULT_MAX_PARALLEL_DOWNLOADS, - max_download_tries: DEFAULT_MAX_DOWNLOAD_TRIES, - wait_between_download_tries: DEFAULT_WAIT_BETWEEN_DOWNLOAD_TRIES, extra: Default::default(), }; diff --git a/crates/tako/src/internal/tests/utils/worker.rs b/crates/tako/src/internal/tests/utils/worker.rs index 6531b3f82..e5ac01a30 100644 --- a/crates/tako/src/internal/tests/utils/worker.rs +++ b/crates/tako/src/internal/tests/utils/worker.rs @@ -1,9 +1,6 @@ use crate::WorkerId; use crate::internal::server::worker::Worker; -use crate::internal::worker::configuration::{ - DEFAULT_MAX_DOWNLOAD_TRIES, DEFAULT_MAX_PARALLEL_DOWNLOADS, - DEFAULT_WAIT_BETWEEN_DOWNLOAD_TRIES, OverviewConfiguration, -}; +use crate::internal::worker::configuration::OverviewConfiguration; use crate::resources::{ResourceDescriptor, ResourceDescriptorItem, ResourceIdMap}; use crate::worker::{ServerLostPolicy, WorkerConfiguration}; use std::time::{Duration, Instant}; @@ -74,9 +71,6 @@ impl WorkerBuilder { idle_timeout: None, time_limit: self.time_limit, on_server_lost: ServerLostPolicy::Stop, - max_parallel_downloads: DEFAULT_MAX_PARALLEL_DOWNLOADS, - max_download_tries: DEFAULT_MAX_DOWNLOAD_TRIES, - wait_between_download_tries: DEFAULT_WAIT_BETWEEN_DOWNLOAD_TRIES, extra: Default::default(), } } diff --git a/crates/tako/src/internal/worker/configuration.rs b/crates/tako/src/internal/worker/configuration.rs index c9a5ed92d..936a83965 100644 --- a/crates/tako/src/internal/worker/configuration.rs +++ b/crates/tako/src/internal/worker/configuration.rs @@ -35,10 +35,6 @@ impl OverviewConfiguration { } } -pub const DEFAULT_MAX_PARALLEL_DOWNLOADS: u32 = 4; -pub const DEFAULT_MAX_DOWNLOAD_TRIES: u32 = 8; -pub const DEFAULT_WAIT_BETWEEN_DOWNLOAD_TRIES: Duration = Duration::from_secs(1); - #[derive(Serialize, Deserialize, Debug, Clone)] pub struct WorkerConfiguration { pub resources: ResourceDescriptor, @@ -52,10 +48,6 @@ pub struct WorkerConfiguration { pub idle_timeout: Option, pub time_limit: Option, pub on_server_lost: ServerLostPolicy, - pub max_parallel_downloads: u32, - pub max_download_tries: u32, - pub wait_between_download_tries: Duration, - pub extra: Map, } diff --git a/tests/output/test_json.py b/tests/output/test_json.py index 240e3c99e..c57686add 100644 --- a/tests/output/test_json.py +++ b/tests/output/test_json.py @@ -51,9 +51,6 @@ def test_print_worker_info(hq_env: HqEnv): "work_dir": str, "group": str, "on_server_lost": "stop", - "max_download_tries": int, - "max_parallel_downloads": int, - "wait_between_download_tries": float, }, "allocation": None, "started": str,