diff --git a/Cargo.toml b/Cargo.toml
index 21c4777..a6ef9ac 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,5 +15,7 @@ structopt = "0.2.18"
 cargo_metadata = "0.8.0"
 log = "0.4.1"
 simple_logger = "1.3.0"
-toml = "0.5.1"
 xdg = "2.1.0"
+serde = { version = "1.0", features = ["derive"] }
+toml = "0.5.1"
+config = "0.11"
diff --git a/src/config.rs b/src/config.rs
new file mode 100644
index 0000000..d409c3a
--- /dev/null
+++ b/src/config.rs
@@ -0,0 +1,56 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Remote {
+    pub host: String,
+    pub user: String,
+    pub ssh_port: u16,
+    pub temp_dir: String,
+}
+
+impl Default for Remote {
+    fn default() -> Self {
+        Self {
+            host: String::new(),
+            user: String::new(),
+            ssh_port: 22,
+            temp_dir: "~/remote-builds".to_string(),
+        }
+    }
+}
+
+impl Remote {
+    pub fn user_host(&self) -> String {
+        format!("{}@{}", self.user, self.host)
+    }
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct Config {
+    pub remote: Remote,
+}
+
+impl Config {
+    pub fn new(project_dir: &std::path::Path) -> Result<Self, config::ConfigError> {
+        let mut conf = config::Config::default();
+
+        conf.merge(config::File::from_str(
+            toml::to_string(&Config::default()).unwrap().as_str(),
+            config::FileFormat::Toml,
+        ))?;
+
+        if let Some(config_file) = xdg::BaseDirectories::with_prefix("cargo-remote")
+            .ok()
+            .and_then(|base| base.find_config_file("cargo-remote.toml"))
+        {
+            conf.merge(config::File::from(config_file))?;
+        }
+
+        let project_config = project_dir.join(".cargo-remote.toml");
+        if project_config.is_file() {
+            conf.merge(config::File::from(project_config))?;
+        }
+
+        conf.try_into()
+    }
+}
diff --git a/src/main.rs b/src/main.rs
index 7409d31..7c0abe4 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,12 +1,13 @@
-use std::path::{Path, PathBuf};
-use std::process::{exit, Command, Stdio};
 use std::collections::hash_map::DefaultHasher;
 use std::hash::{Hash, Hasher};
+use std::path::PathBuf;
+use std::process::{exit, Command, Stdio};
 use structopt::StructOpt;
-use toml::Value;
 
 use log::{error, info, warn};
 
+mod config;
+
 const PROGRESS_FLAG: &str = "--info=progress2";
 
 #[derive(StructOpt, Debug)]
@@ -80,33 +81,6 @@ enum Opts {
     },
 }
 
-/// Tries to parse the file [`config_path`]. Logs warnings and returns [`None`] if errors occur
-/// during reading or parsing, [`Some(Value)`] otherwise.
-fn config_from_file(config_path: &Path) -> Option<Value> {
-    let config_file = std::fs::read_to_string(config_path)
-        .map_err(|e| {
-            warn!(
-                "Can't parse config file '{}' (error: {})",
-                config_path.to_string_lossy(),
-                e
-            );
-        })
-        .ok()?;
-
-    let value = config_file
-        .parse::<Value>()
-        .map_err(|e| {
-            warn!(
-                "Can't parse config file '{}' (error: {})",
-                config_path.to_string_lossy(),
-                e
-            );
-        })
-        .ok()?;
-
-    Some(value)
-}
-
 fn main() {
     simple_logger::init().unwrap();
 
@@ -130,31 +104,20 @@ fn main() {
     let project_dir = project_metadata.workspace_root;
     info!("Project dir: {:?}", project_dir);
 
-    let configs = vec![
-        config_from_file(&project_dir.join(".cargo-remote.toml")),
-        xdg::BaseDirectories::with_prefix("cargo-remote")
-            .ok()
-            .and_then(|base| base.find_config_file("cargo-remote.toml"))
-            .and_then(|p: PathBuf| config_from_file(&p)),
-    ];
-
-    // TODO: move Opts::Remote fields into own type and implement complete_from_config(&mut self, config: &Value)
-    let build_server = remote
-        .or_else(|| {
-            configs
-                .into_iter()
-                .flat_map(|config| config.and_then(|c| c["remote"].as_str().map(String::from)))
-                .next()
-        })
-        .unwrap_or_else(|| {
-            error!("No remote build server was defined (use config file or --remote flag)");
+    let conf = match config::Config::new(&project_dir) {
+        Ok(conf) => conf,
+        Err(error) => {
+            error!("{}", error);
             exit(-3);
-        });
+        }
+    };
+
+    let build_server = remote.unwrap_or_else(|| conf.remote.user_host());
 
     // generate a unique build path by using the hashed project dir as folder on the remote machine
     let mut hasher = DefaultHasher::new();
     project_dir.hash(&mut hasher);
-    let build_path = format!("~/remote-builds/{}/", hasher.finish());
+    let build_path = format!("{}/{}/", conf.remote.temp_dir, hasher.finish());
 
     info!("Transferring sources to build server.");
     // transfer project to build server
@@ -163,6 +126,7 @@ fn main() {
         .arg("-a".to_owned())
         .arg("--delete")
         .arg("--compress")
+        .arg(format!("-e ssh -p {}", conf.remote.ssh_port))
         .arg("--info=progress2")
         .arg("--exclude")
         .arg("target");
@@ -199,6 +163,7 @@ fn main() {
 
     info!("Starting build process.");
     let output = Command::new("ssh")
+        .arg(format!("-p {}", conf.remote.ssh_port))
         .arg("-t")
         .arg(&build_server)
         .arg(build_command)
@@ -218,9 +183,17 @@ fn main() {
             .arg("-a")
             .arg("--delete")
             .arg("--compress")
+            .arg(format!("-e ssh -p {}", conf.remote.ssh_port))
             .arg("--info=progress2")
-            .arg(format!("{}:{}/target/{}", build_server, build_path, file_name))
-            .arg(format!("{}/target/{}", project_dir.to_string_lossy(), file_name))
+            .arg(format!(
+                "{}:{}/target/{}",
+                build_server, build_path, file_name
+            ))
+            .arg(format!(
+                "{}/target/{}",
+                project_dir.to_string_lossy(),
+                file_name
+            ))
             .stdout(Stdio::inherit())
             .stderr(Stdio::inherit())
             .stdin(Stdio::inherit())
@@ -240,6 +213,7 @@ fn main() {
             .arg("-a")
             .arg("--delete")
             .arg("--compress")
+            .arg(format!("-e ssh -p {}", conf.remote.ssh_port))
             .arg("--info=progress2")
             .arg(format!("{}:{}/Cargo.lock", build_server, build_path))
             .arg(format!("{}/Cargo.lock", project_dir.to_string_lossy()))