Why is this counter going down when the tab is in the background?

I have a counter that subtracts the delta each from and adds the update frequency from the server (at the same frequency) . Here is a simplified version of my code:

(function loop(now) {

var now = Date.now();
delta = now - Time;
		
Time = now;
	
timeElapsed -= delta;	

if(frameAvg == null){
frameAvg = delta;
}
frameAvg = (frameAvg + delta) / 2;
	



counter += frameAvg;

if(counter > updateFrequency){
timeElapsed += updateFrequency;
counter -= updateFrequency;	
}

console.log(timeElapsed);

requestAnimationFrame(loop)
			
	})(0);	

When I load the page and take a look at console it is fine, timeElapsed is around 300. But if I switch tabs for a few minutes timeElapsed will then be far into the negative numbers (around -3000 if you leave it for a few minutes). I don't understand what the problem is, how can I fix it?