feat: emit envd logs through orchestrator logger

packages/api/internal/cfg/model.go

@@ -73,6 +73,8 @@ type Config struct {
 	SandboxStorageBackend string `env:"SANDBOX_STORAGE_BACKEND" envDefault:"memory"`
 
 	DomainName string `env:"DOMAIN_NAME" envDefault:""`
+
+	LogsCollectorAddress string `env:"LOGS_COLLECTOR_ADDRESS" envDefault:""`
 }
 
 type JWTSigningKey any

packages/api/main.go

@@ -251,6 +251,11 @@ func run() int {
 	serviceInstanceID := uuid.New().String()
 	nodeID := env.GetNodeID()
 
+	config, err := cfg.Parse()
+	if err != nil {
+		logger.L().Fatal(ctx, "Error parsing config", zap.Error(err))
+	}
+
 	tel, err := telemetry.New(ctx, nodeID, serviceName, commitSHA, serviceVersion, serviceInstanceID)
 	if err != nil {
 		logger.L().Fatal(ctx, "failed to create metrics exporter", zap.Error(err))
@@ -279,7 +284,7 @@ func run() int {
 		sbxlogger.SandboxLoggerConfig{
 			ServiceName:      serviceName,
 			IsInternal:       false,
-			CollectorAddress: env.LogsCollectorAddress(),
+			CollectorAddress: config.LogsCollectorAddress,
 		},
 	)
 	defer sbxLoggerExternal.Sync()
@@ -291,7 +296,7 @@ func run() int {
 		sbxlogger.SandboxLoggerConfig{
 			ServiceName:      serviceName,
 			IsInternal:       true,
-			CollectorAddress: env.LogsCollectorAddress(),
+			CollectorAddress: config.LogsCollectorAddress,
 		},
 	)
 	defer sbxLoggerInternal.Sync()
@@ -310,11 +315,6 @@ func run() int {
 		expectedMigration = 0
 	}
 
-	config, err := cfg.Parse()
-	if err != nil {
-		logger.L().Fatal(ctx, "Error parsing config", zap.Error(err))
-	}
-
 	err = sqlcdb.CheckMigrationVersion(ctx, config.PostgresConnectionString, expectedMigration)
 	if err != nil {
 		l.Fatal(ctx, "failed to check migration version", zap.Error(err))

packages/envd/internal/api/compose_test.go

@@ -32,7 +32,7 @@ func newComposeTestAPI(t *testing.T) (*API, *user.User) {
 		User:    currentUser.Username,
 	}
 
-	return New(&logger, defaults, nil, false), currentUser
+	return New(logger, defaults, nil, false), currentUser
 }
 
 func writeSourceFile(t *testing.T, dir string, name string, data []byte) string {

packages/envd/internal/api/download_test.go

@@ -95,7 +95,7 @@ func TestGetFilesContentDisposition(t *testing.T) {
 				EnvVars: utils.NewMap[string, string](),
 				User:    currentUser.Username,
 			}
-			api := New(&logger, defaults, nil, false)
+			api := New(logger, defaults, nil, false)
 
 			// Create request and response recorder
 			req := httptest.NewRequest(http.MethodGet, "/files?path="+url.QueryEscape(tempFile), nil)
@@ -144,7 +144,7 @@ func TestGetFilesContentDispositionWithNestedPath(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	// Create request and response recorder
 	req := httptest.NewRequest(http.MethodGet, "/files?path="+url.QueryEscape(tempFile), nil)
@@ -187,7 +187,7 @@ func TestGetFiles_GzipEncoding_ExplicitIdentityOffWithRange(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	// Create request and response recorder
 	req := httptest.NewRequest(http.MethodGet, "/files?path="+url.QueryEscape(tempFile), nil)
@@ -228,7 +228,7 @@ func TestGetFiles_GzipDownload(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	req := httptest.NewRequest(http.MethodGet, "/files?path="+url.QueryEscape(tempFile), nil)
 	req.Header.Set("Accept-Encoding", "gzip")
@@ -293,7 +293,7 @@ func TestPostFiles_GzipUpload(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	req := httptest.NewRequest(http.MethodPost, "/files?path="+url.QueryEscape(destPath), &gzBuf)
 	req.Header.Set("Content-Type", mpWriter.FormDataContentType())
@@ -333,7 +333,7 @@ func TestPostFiles_RawBodyUpload(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	req := httptest.NewRequest(http.MethodPost, "/files?path="+url.QueryEscape(destPath), bytes.NewReader(originalContent))
 	req.Header.Set("Content-Type", "application/octet-stream")
@@ -371,7 +371,7 @@ func TestPostFiles_RawBodyUploadCreatesDirectories(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	req := httptest.NewRequest(http.MethodPost, "/files?path="+url.QueryEscape(destPath), bytes.NewReader(originalContent))
 	req.Header.Set("Content-Type", "application/octet-stream")
@@ -404,7 +404,7 @@ func TestPostFiles_RawBodyUploadRequiresPath(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	req := httptest.NewRequest(http.MethodPost, "/files", bytes.NewReader([]byte("some content")))
 	req.Header.Set("Content-Type", "application/octet-stream")
@@ -439,7 +439,7 @@ func TestPostFiles_RawBodyUploadOverwritesExisting(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	req := httptest.NewRequest(http.MethodPost, "/files?path="+url.QueryEscape(destPath), bytes.NewReader(newContent))
 	req.Header.Set("Content-Type", "application/octet-stream")
@@ -485,7 +485,7 @@ func TestPostFiles_RawBodyGzipUpload(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	req := httptest.NewRequest(http.MethodPost, "/files?path="+url.QueryEscape(destPath), &gzBuf)
 	req.Header.Set("Content-Type", "application/octet-stream")
@@ -519,7 +519,7 @@ func TestPostFiles_UnsupportedContentType(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	tempDir := t.TempDir()
 	destPath := filepath.Join(tempDir, "test.txt")
@@ -565,7 +565,7 @@ func TestPostFiles_MultipartStillWorksWithoutContentType(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	req := httptest.NewRequest(http.MethodPost, "/files?path="+url.QueryEscape(destPath), &multipartBuf)
 	req.Header.Set("Content-Type", mpWriter.FormDataContentType())
@@ -623,7 +623,7 @@ func TestGzipUploadThenGzipDownload(t *testing.T) {
 		EnvVars: utils.NewMap[string, string](),
 		User:    currentUser.Username,
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 
 	uploadReq := httptest.NewRequest(http.MethodPost, "/files?path="+url.QueryEscape(destPath), &gzBuf)
 	uploadReq.Header.Set("Content-Type", mpWriter.FormDataContentType())

packages/envd/internal/api/init.go

@@ -153,7 +153,7 @@ func (a *API) PostInit(w http.ResponseWriter, r *http.Request) {
 	go func() { //nolint:contextcheck // TODO: fix this later
 		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 		defer cancel()
-		host.PollForMMDSOpts(ctx, a.mmdsChan, a.defaults.EnvVars)
+		host.PollForMMDSOpts(ctx, a.logger, a.mmdsChan, a.defaults.EnvVars)
 	}()
 
 	w.Header().Set("Cache-Control", "no-store")

packages/envd/internal/api/init_test.go

@@ -142,7 +142,7 @@ func newTestAPI(accessToken *SecureToken, mmdsClient MMDSClient) *API {
 	defaults := &execcontext.Defaults{
 		EnvVars: utils.NewMap[string, string](),
 	}
-	api := New(&logger, defaults, nil, false)
+	api := New(logger, defaults, nil, false)
 	if accessToken != nil {
 		api.accessToken.TakeFrom(accessToken)
 	}

packages/envd/internal/api/store.go

@@ -27,7 +27,7 @@ func (c *DefaultMMDSClient) GetAccessTokenHash(ctx context.Context) (string, err
 
 type API struct {
 	isNotFC     bool
-	logger      *zerolog.Logger
+	logger      zerolog.Logger
 	accessToken *SecureToken
 	defaults    *execcontext.Defaults
 
@@ -39,7 +39,7 @@ type API struct {
 	initLock    sync.Mutex
 }
 
-func New(l *zerolog.Logger, defaults *execcontext.Defaults, mmdsChan chan *host.MMDSOpts, isNotFC bool) *API {
+func New(l zerolog.Logger, defaults *execcontext.Defaults, mmdsChan chan *host.MMDSOpts, isNotFC bool) *API {
 	return &API{
 		logger:      l,
 		defaults:    defaults,

packages/envd/internal/host/mmds.go

@@ -11,6 +11,8 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/rs/zerolog"
+
 	"github.com/e2b-dev/infra/packages/envd/internal/utils"
 )
 
@@ -134,7 +136,7 @@ func GetAccessTokenHashFromMMDS(ctx context.Context) (string, error) {
 	return opts.AccessTokenHash, nil
 }
 
-func PollForMMDSOpts(ctx context.Context, mmdsChan chan<- *MMDSOpts, envVars *utils.Map[string, string]) {
+func PollForMMDSOpts(ctx context.Context, logger zerolog.Logger, mmdsChan chan<- *MMDSOpts, envVars *utils.Map[string, string]) {
 	httpClient := &http.Client{}
 	defer httpClient.CloseIdleConnections()
 
@@ -144,20 +146,20 @@ func PollForMMDSOpts(ctx context.Context, mmdsChan chan<- *MMDSOpts, envVars *ut
 	for {
 		select {
 		case <-ctx.Done():
-			fmt.Fprintf(os.Stderr, "context cancelled while waiting for mmds opts")
+			logger.Error().Msg("context cancelled while waiting for mmds opts")
 
 			return
 		case <-ticker.C:
 			token, err := getMMDSToken(ctx, httpClient)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "error getting mmds token: %v\n", err)
+				logger.Error().Err(err).Msg("error getting mmds token")
 
 				continue
 			}
 
 			mmdsOpts, err := getMMDSOpts(ctx, httpClient, token)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "error getting mmds opts: %v\n", err)
+				logger.Error().Err(err).Msg("error getting mmds opts")
 
 				continue
 			}
@@ -166,10 +168,10 @@ func PollForMMDSOpts(ctx context.Context, mmdsChan chan<- *MMDSOpts, envVars *ut
 			envVars.Store("E2B_TEMPLATE_ID", mmdsOpts.TemplateID)
 
 			if err := os.WriteFile(filepath.Join(E2BRunDir, ".E2B_SANDBOX_ID"), []byte(mmdsOpts.SandboxID), 0o666); err != nil {
-				fmt.Fprintf(os.Stderr, "error writing sandbox ID file: %v\n", err)
+				logger.Error().Err(err).Msg("error writing sandbox ID file")
 			}
 			if err := os.WriteFile(filepath.Join(E2BRunDir, ".E2B_TEMPLATE_ID"), []byte(mmdsOpts.TemplateID), 0o666); err != nil {
-				fmt.Fprintf(os.Stderr, "error writing template ID file: %v\n", err)
+				logger.Error().Err(err).Msg("error writing template ID file")
 			}
 
 			if mmdsOpts.LogsCollectorAddress != "" {

packages/envd/internal/services/process/handler/handler.go

@@ -221,7 +221,7 @@ func New(
 				}
 
 				if readErr != nil {
-					fmt.Fprintf(os.Stderr, "error reading from pty: %s\n", readErr)
+					logger.Error().Err(readErr).Msg("error reading from pty")
 
 					break
 				}
@@ -265,7 +265,7 @@ func New(
 				}
 
 				if readErr != nil {
-					fmt.Fprintf(os.Stderr, "error reading from stdout: %s\n", readErr)
+					logger.Error().Err(readErr).Msg("error reading from stdout")
 
 					break
 				}
@@ -307,7 +307,7 @@ func New(
 				}
 
 				if readErr != nil {
-					fmt.Fprintf(os.Stderr, "error reading from stderr: %s\n", readErr)
+					logger.Error().Err(readErr).Msg("error reading from stderr")
 
 					break
 				}

packages/envd/main.go

@@ -15,6 +15,7 @@ import (
 	connectcors "connectrpc.com/cors"
 	"github.com/go-chi/chi/v5"
 	"github.com/rs/cors"
+	"github.com/rs/zerolog"
 
 	"github.com/e2b-dev/infra/packages/envd/internal/api"
 	"github.com/e2b-dev/infra/packages/envd/internal/execcontext"
@@ -147,46 +148,47 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	if err := os.MkdirAll(host.E2BRunDir, 0o755); err != nil {
-		fmt.Fprintf(os.Stderr, "error creating E2B run directory: %v\n", err)
-	}
-
 	defaults := &execcontext.Defaults{
 		User:    defaultUser,
 		EnvVars: utils.NewMap[string, string](),
 	}
+
+	mmdsChan := make(chan *host.MMDSOpts, 1)
+	defer close(mmdsChan)
+	l := logs.NewLogger(ctx, isNotFC, mmdsChan)
+
+	if err := os.MkdirAll(host.E2BRunDir, 0o755); err != nil {
+		l.Error().Err(err).Msg("error creating E2B run directory")
+	}
+
 	isFCBoolStr := strconv.FormatBool(!isNotFC)
 	defaults.EnvVars.Store("E2B_SANDBOX", isFCBoolStr)
 	if err := os.WriteFile(filepath.Join(host.E2BRunDir, ".E2B_SANDBOX"), []byte(isFCBoolStr), 0o444); err != nil {
-		fmt.Fprintf(os.Stderr, "error writing sandbox file: %v\n", err)
+		l.Error().Err(err).Msg("error writing sandbox file")
 	}
 
-	mmdsChan := make(chan *host.MMDSOpts, 1)
-	defer close(mmdsChan)
 	if !isNotFC {
-		go host.PollForMMDSOpts(ctx, mmdsChan, defaults.EnvVars)
+		go host.PollForMMDSOpts(ctx, l.With().Str("logger", "mmds-poller").Logger(), mmdsChan, defaults.EnvVars)
 	}
 
-	l := logs.NewLogger(ctx, isNotFC, mmdsChan)
-
 	m := chi.NewRouter()
 
 	envLogger := l.With().Str("logger", "envd").Logger()
 	fsLogger := l.With().Str("logger", "filesystem").Logger()
 	filesystemRpc.Handle(m, &fsLogger, defaults)
 
-	cgroupManager := createCgroupManager()
+	cgroupManager := createCgroupManager(l)
 	defer func() {
 		err := cgroupManager.Close()
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "failed to close cgroup manager: %v\n", err)
+			l.Error().Err(err).Msg("failed to close cgroup manager")
 		}
 	}()
 
 	processLogger := l.With().Str("logger", "process").Logger()
 	processService := processRpc.Handle(m, &processLogger, defaults, cgroupManager)
 
-	service := api.New(&envLogger, defaults, mmdsChan, isNotFC)
+	service := api.New(envLogger, defaults, mmdsChan, isNotFC)
 	handler := api.HandlerFromMux(service, m)
 	middleware := authn.NewMiddleware(permissions.AuthenticateUsername)
 
@@ -241,17 +243,17 @@ func main() {
 	}
 }
 
-func createCgroupManager() (m cgroups.Manager) {
+func createCgroupManager(logger *zerolog.Logger) (m cgroups.Manager) {
 	defer func() {
 		if m == nil {
-			fmt.Fprintf(os.Stderr, "falling back to no-op cgroup manager\n")
+			logger.Warn().Msg("falling back to no-op cgroup manager")
 			m = cgroups.NewNoopManager()
 		}
 	}()
 
 	metrics, err := host.GetMetrics()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to calculate host metrics: %v\n", err)
+		logger.Error().Err(err).Msg("failed to calculate host metrics")
 
 		return nil
 	}
@@ -284,7 +286,7 @@ func createCgroupManager() (m cgroups.Manager) {
 
 	mgr, err := cgroups.NewCgroup2Manager(opts...)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to create cgroup2 manager: %v\n", err)
+		logger.Error().Err(err).Msg("failed to create cgroup2 manager")
 
 		return nil
 	}