diff --git a/swarm.go b/swarm.go
index f85ef5234b8ba17f79391e9b9d8c59d48667ecbb..fe4e1ee978d205c788dce3a0c62aab1fa5a48894 100644
--- a/swarm.go
+++ b/swarm.go
@@ -112,9 +112,13 @@ func NewSwarm(ctx context.Context, local peer.ID, peers peerstore.Peerstore, bwc
 
 	s.dsync = NewDialSync(s.doDial)
 	s.limiter = newDialLimiter(s.dialAddr)
-	s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown)
+	s.proc = goprocessctx.WithContext(ctx)
 	s.ctx = goprocessctx.OnClosingContext(s.proc)
 
+	// Set teardown after setting the context/process so we don't start the
+	// teardown process early.
+	s.proc.SetTeardown(s.teardown)
+
 	return s
 }