diff --git a/packages/orchestrator/pkg/sandbox/build/cache.go b/packages/orchestrator/pkg/sandbox/build/cache.go index 812e274e7c..9b08175694 100644 --- a/packages/orchestrator/pkg/sandbox/build/cache.go +++ b/packages/orchestrator/pkg/sandbox/build/cache.go @@ -8,6 +8,8 @@ import ( "time" "github.com/jellydator/ttlcache/v3" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" "go.uber.org/zap" "golang.org/x/sys/unix" @@ -15,9 +17,17 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/pkg/units" "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" "github.com/e2b-dev/infra/packages/shared/pkg/logger" + "github.com/e2b-dev/infra/packages/shared/pkg/utils" ) -var fallbackDiffSize = units.MBToBytes(100) +var ( + fallbackDiffSize = units.MBToBytes(100) + + meter = otel.Meter("github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/build") + residenceDurationMetric = utils.Must(meter.Int64Histogram("orchestrator.build.cache.residence_duration", + metric.WithDescription("How long a diff was kept in the local build cache before eviction"), + metric.WithUnit("s"))) +) type deleteDiff struct { size int64 @@ -37,6 +47,8 @@ type DiffStore struct { pdSizes map[DiffStoreKey]*deleteDiff pdMu sync.RWMutex pdDelay time.Duration + + insertionTimes sync.Map // map[DiffStoreKey]time.Time — tracks when each diff was cached } func NewDiffStore( @@ -65,7 +77,13 @@ func NewDiffStore( } cache.OnEviction(func(ctx context.Context, _ ttlcache.EvictionReason, item *ttlcache.Item[DiffStoreKey, Diff]) { + if insertedAt, ok := ds.insertionTimes.LoadAndDelete(item.Key()); ok { + duration := time.Since(insertedAt.(time.Time)) + residenceDurationMetric.Record(ctx, int64(duration.Seconds())) + } + buildData := item.Value() + // buildData will be deleted by calling buildData.Close() defer ds.resetDelete(item.Key()) @@ -110,6 +128,8 @@ func (s *DiffStore) Get(ctx context.Context, diff Diff) (Diff, error) { } if !found { + s.insertionTimes.Store(diff.CacheKey(), time.Now()) + err := diff.Init(ctx) if err != nil { return nil, fmt.Errorf("failed to init source: %w", err) @@ -122,6 +142,7 @@ func (s *DiffStore) Get(ctx context.Context, diff Diff) (Diff, error) { func (s *DiffStore) Add(d Diff) { s.resetDelete(d.CacheKey()) s.cache.Set(d.CacheKey(), d, ttlcache.DefaultTTL) + s.insertionTimes.LoadOrStore(d.CacheKey(), time.Now()) } func (s *DiffStore) Has(d Diff) bool {