Mobile proxies for Avito

import csv
import random
import time
from datetime import datetime

import requests

PROXY = "http://LOGIN:PASSWORD@HOST:PORT"
ROTATE_URL = "https://mobihub.biz/api/v1/proxies/PROXY_ID/ip"
URLS_FILE = "listings.txt"
REPORT_FILE = "listings_report.csv"
PAUSE = (6, 14)
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/126.0 Mobile Safari/537.36",
    "Accept-Language": "ru-RU,ru;q=0.9",
}

proxies = {"http": PROXY, "https": PROXY}


def fetch(url):
    response = requests.get(url, headers=HEADERS, proxies=proxies, timeout=25)
    return response.status_code, response.url


def check(url):
    try:
        status, final_url = fetch(url)
        if status in (403, 429):
            requests.get(ROTATE_URL, timeout=15)
            time.sleep(40)
            status, final_url = fetch(url)
    except requests.RequestException as error:
        return "error", str(error)[:80]

    if status == 200 and final_url.rstrip("/") == url.rstrip("/"):
        return "active", final_url
    if status == 200:
        return "redirected", final_url
    if status in (404, 410):
        return "removed", final_url
    return f"http {status}", final_url


def main():
    with open(URLS_FILE) as f:
        urls = [line.strip() for line in f if line.strip()]

    with open(REPORT_FILE, "w", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(["checked_at", "url", "state", "details"])
        for url in urls:
            state, details = check(url)
            writer.writerow([datetime.now().isoformat(timespec="seconds"), url, state, details])
            f.flush()
            print(f"{state:<12} {url}")
            time.sleep(random.uniform(*PAUSE))


if __name__ == "__main__":
    main()