voyager-api/ScrapperAPI/Utils/DomainRateLimiter.cs

using System.Collections.Concurrent;
using ScrapperAPI.Interfaces;

namespace ScrapperAPI.Utils;

public sealed class DomainRateLimiter : IDomainRateLimiter
{
    private readonly ConcurrentDictionary<string, HostLimiter> _hosts = new();
    private readonly int _minDelayMs;

    public DomainRateLimiter(int minDelayMs)
    {
        _minDelayMs = Math.Max(0, minDelayMs);
    }

    public async Task WaitAsync(string host, CancellationToken ct)
    {
        if (_minDelayMs == 0) return;

        var limiter = _hosts.GetOrAdd(host, _ => new HostLimiter());
        
        await limiter.Gate.WaitAsync(ct);
        try
        {
            var now = DateTimeOffset.UtcNow;
            var next = limiter.NextAllowedUtc;

            if (next > now)
            {
                var delay = next - now;
                await Task.Delay(delay, ct);
                now = DateTimeOffset.UtcNow;
            }

            limiter.NextAllowedUtc = now.AddMilliseconds(_minDelayMs);
        }
        finally
        {
            limiter.Gate.Release();
        }
    }

    private sealed class HostLimiter
    {
        public SemaphoreSlim Gate { get; } = new(1, 1);
        public DateTimeOffset NextAllowedUtc { get; set; } = DateTimeOffset.MinValue;
    }
}