From 4e5eed599c58a264ca36f0d3153a43df78abc231 Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Tue, 6 May 2025 15:26:32 +0200 Subject: [PATCH] get-lore-repositories: scan_for_repo: add basic retries Use requests.Session with HTTPAdapter(max_retries=3) to retry temporary network or server failures up to 3 times. This improves robustness with minimal overhead. Also include a custom User-Agent header to identify the script. Signed-off-by: Anders Roxell --- get-lore-repositories.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/get-lore-repositories.py b/get-lore-repositories.py index e6ea2952f3fe..24eba823d19f 100755 --- a/get-lore-repositories.py +++ b/get-lore-repositories.py @@ -12,6 +12,7 @@ import re import requests import subprocess as sp from pathlib import Path +from requests.adapters import HTTPAdapter from urllib.parse import urlparse import os @@ -89,7 +90,10 @@ def scan_for_repo(url, all_repos=False): full_url = f"{url}/_/text/mirror/" logger.debug(f"{full_url=}") headers = {'User-Agent': 'LKFT-get-lore-repositories'} - resp = requests.get(full_url, headers=headers) + session = requests.Session() + session.mount("http://", HTTPAdapter(max_retries=3)) + session.mount("https://", HTTPAdapter(max_retries=3)) + resp = session.get(full_url, headers=headers) repos = [] pattern = re.compile(r'href="(https?://[^"]*/\d+)"') if resp.status_code >= 200 and resp.status_code < 300: -- 2.47.2