diff --git a/logtail/flusher.py b/logtail/flusher.py index 2c7d16d..4c9f725 100644 --- a/logtail/flusher.py +++ b/logtail/flusher.py @@ -30,6 +30,7 @@ def __init__(self, upload: Callable[[list[Any]], Any], pipe: 'queue.Queue[Any]', self.should_run = True self._flushing = False self._clean = True + self._reported_rejections: set[int] = set() def run(self) -> None: while self.should_run: @@ -96,11 +97,29 @@ def step(self) -> None: if _should_retry(response.status_code): reason = getattr(response, 'exception', 'HTTP {}'.format(response.status_code)) print('Failed to send logs to Better Stack after {} retries: {}'.format(len(RETRY_SCHEDULE), reason)) + elif not 200 <= response.status_code < 300: + self._report_rejected_batch(len(frame), response) self._clean = True if shutdown and self.pipe.empty(): self.should_run = False + def _report_rejected_batch(self, record_count: int, response: Any) -> None: + # A batch is flushed every second, so a line per rejected batch would flood the output + # of an app with a wrong source token. + if response.status_code in self._reported_rejections: + return + self._reported_rejections.add(response.status_code) + + status = 'HTTP {} {}'.format(response.status_code, getattr(response, 'reason', None) or '').strip() + body = ' '.join(getattr(response, 'text', '')[:200].split()) + print("Better Stack rejected {} log {} with {}{}. Further rejections with this status won't be reported.".format( + record_count, + 'record' if record_count == 1 else 'records', + status, + ': {}'.format(body) if body else '', + )) + def flush(self, timeout: Optional[float] = None) -> bool: """Block until the worker has drained the queue. diff --git a/tests/test_flusher.py b/tests/test_flusher.py index b203372..231ee26 100644 --- a/tests/test_flusher.py +++ b/tests/test_flusher.py @@ -4,6 +4,7 @@ import requests import time import threading +import types import unittest from unittest.mock import patch @@ -201,6 +202,121 @@ def uploader(frame): mock_print.assert_called_once_with('Failed to send logs to Better Stack after 3 retries: network is down') + def _response(self, status_code, reason=None, content=b''): + response = requests.models.Response() + response.status_code = status_code + response.reason = reason + response._content = content + return response + + def _fill(self, pipe): + for log in range(self.buffer_capacity): + pipe.put(log, block=False) + + @patch('logtail.flusher.time.sleep') + def test_step_reports_a_rejected_batch_with_the_server_reason(self, _mock_sleep): + def uploader(frame): + return self._response(401, 'Unauthorized', b'{"errors":"Invalid source token"}') + + pipe, _, fw = self._setup_worker(uploader) + self._fill(pipe) + + with patch('builtins.print') as mock_print: + fw.step() + + self.assertTrue(fw._clean) + mock_print.assert_called_once_with( + 'Better Stack rejected 5 log records with HTTP 401 Unauthorized: {"errors":"Invalid source token"}. ' + "Further rejections with this status won't be reported." + ) + + @patch('logtail.flusher.time.sleep') + def test_step_does_not_retry_a_rejected_batch(self, mock_sleep): + self.uploader_calls = 0 + + def uploader(frame): + self.uploader_calls += 1 + return self._response(401, 'Unauthorized', b'{"errors":"Invalid source token"}') + + pipe, _, fw = self._setup_worker(uploader) + self._fill(pipe) + + with patch('builtins.print'): + fw.step() + + self.assertEqual(self.uploader_calls, 1) + mock_sleep.assert_not_called() + + @patch('logtail.flusher.time.sleep') + def test_step_reports_each_rejecting_status_once(self, _mock_sleep): + responses = iter([ + self._response(401, 'Unauthorized', b'{"errors":"Invalid source token"}'), + self._response(401, 'Unauthorized', b'{"errors":"Invalid source token"}'), + self._response(422, 'Unprocessable Entity'), + self._response(202, 'Accepted'), + self._response(422, 'Unprocessable Entity'), + self._response(401, 'Unauthorized', b'{"errors":"Invalid source token"}'), + ]) + + pipe, _, fw = self._setup_worker(lambda frame: next(responses)) + + with patch('builtins.print') as mock_print: + for _ in range(6): + self._fill(pipe) + fw.step() + + self.assertEqual(mock_print.call_args_list, [ + mock.call( + 'Better Stack rejected 5 log records with HTTP 401 Unauthorized: {"errors":"Invalid source token"}. ' + "Further rejections with this status won't be reported." + ), + mock.call( + 'Better Stack rejected 5 log records with HTTP 422 Unprocessable Entity. ' + "Further rejections with this status won't be reported." + ), + ]) + + def test_step_does_not_report_an_accepted_batch(self): + pipe, _, fw = self._setup_worker(lambda frame: self._response(202, 'Accepted')) + self._fill(pipe) + + with patch('builtins.print') as mock_print: + fw.step() + + mock_print.assert_not_called() + + def test_step_reports_a_rejected_single_record_with_the_start_of_an_html_reason_on_one_line(self): + self.buffer_capacity = 1 + content = ( + b'\r\n