diff --git a/crates/crates_io_smoke_test/src/cargo.rs b/crates/crates_io_smoke_test/src/cargo.rs
index 73fca50344f..539406d0cea 100644
--- a/crates/crates_io_smoke_test/src/cargo.rs
+++ b/crates/crates_io_smoke_test/src/cargo.rs
@@ -15,10 +15,22 @@ pub async fn new_lib(parent_path: &Path, name: &str) -> anyhow::Result<()> {
         .map_err(Into::into)
 }
 
+#[allow(unstable_name_collisions)]
+pub async fn package(project_path: &Path) -> anyhow::Result<()> {
+    Command::new("cargo")
+        .args(["package"])
+        .current_dir(project_path)
+        .env("CARGO_TERM_COLOR", "always")
+        .status()
+        .await?
+        .exit_ok()
+        .map_err(Into::into)
+}
+
 #[allow(unstable_name_collisions)]
 pub async fn publish(project_path: &Path, token: &SecretString) -> anyhow::Result<()> {
     Command::new("cargo")
-        .args(["publish", "--registry", "staging", "--allow-dirty"])
+        .args(["publish", "--registry", "staging"])
         .current_dir(project_path)
         .env("CARGO_TERM_COLOR", "always")
         .env(
diff --git a/crates/crates_io_smoke_test/src/git.rs b/crates/crates_io_smoke_test/src/git.rs
new file mode 100644
index 00000000000..801c6cde19c
--- /dev/null
+++ b/crates/crates_io_smoke_test/src/git.rs
@@ -0,0 +1,25 @@
+use crate::exit_status_ext::ExitStatusExt;
+use std::path::Path;
+use tokio::process::Command;
+
+#[allow(unstable_name_collisions)]
+pub async fn add_all(project_path: &Path) -> anyhow::Result<()> {
+    Command::new("git")
+        .args(["add", "--all"])
+        .current_dir(project_path)
+        .status()
+        .await?
+        .exit_ok()
+        .map_err(Into::into)
+}
+
+#[allow(unstable_name_collisions)]
+pub async fn commit(project_path: &Path, message: &str) -> anyhow::Result<()> {
+    Command::new("git")
+        .args(["commit", "--message", message])
+        .current_dir(project_path)
+        .status()
+        .await?
+        .exit_ok()
+        .map_err(Into::into)
+}
diff --git a/crates/crates_io_smoke_test/src/main.rs b/crates/crates_io_smoke_test/src/main.rs
index 710c1e101e0..891ad6d9baa 100644
--- a/crates/crates_io_smoke_test/src/main.rs
+++ b/crates/crates_io_smoke_test/src/main.rs
@@ -1,6 +1,7 @@
 mod api;
 mod cargo;
 mod exit_status_ext;
+mod git;
 
 #[macro_use]
 extern crate tracing;
@@ -52,27 +53,34 @@ async fn main() -> anyhow::Result<()> {
     let old_version = krate.max_version;
     let mut new_version = old_version.clone();
 
-    if options.skip_publish {
-        info!("Skipping publish step");
-    } else {
-        new_version.patch += 1;
-        info!(%old_version, %new_version, "Calculated new version number");
+    new_version.patch += 1;
+    info!(%old_version, %new_version, "Calculated new version number");
 
-        info!("Creating temporary working folder…");
-        let tempdir = tempdir().context("Failed to create temporary working folder")?;
-        debug!(tempdir.path = %tempdir.path().display());
+    info!("Creating temporary working folder…");
+    let tempdir = tempdir().context("Failed to create temporary working folder")?;
+    debug!(tempdir.path = %tempdir.path().display());
 
-        info!("Creating `{}` project…", options.crate_name);
-        let project_path = create_project(tempdir.path(), &options.crate_name, &new_version)
+    info!("Creating `{}` project…", options.crate_name);
+    let project_path = create_project(tempdir.path(), &options.crate_name, &new_version)
+        .await
+        .context("Failed to create project")?;
+
+    if options.skip_publish {
+        info!("Packaging crate file…");
+        cargo::package(&project_path)
             .await
-            .context("Failed to create project")?;
+            .context("Failed to run `cargo package`")?;
 
+        info!("Skipping publish step");
+    } else {
         info!("Publishing to staging.crates.io…");
         cargo::publish(&project_path, &options.token)
             .await
             .context("Failed to run `cargo publish`")?;
     }
 
+    drop(tempdir);
+
     let version = new_version;
     info!(%version, "Checking staging.crates.io API for the new version…");
 
@@ -217,5 +225,14 @@ description = "test crate"
             .context("Failed to write `README.md` file content")?;
     }
 
+    info!("Creating initial git commit…");
+    git::add_all(&project_path)
+        .await
+        .context("Failed to add initial changes to git")?;
+
+    git::commit(&project_path, "initial commit")
+        .await
+        .context("Failed to commit initial changes")?;
+
     Ok(project_path)
 }