Proxies for price monitoring that shops do not notice

#!/usr/bin/env python3
"""Price monitor for Advanced.name proxies.

Fetches product pages through proxies from the buyer's country and pulls the
price out of each page. One request per product goes through the next proxy, so
a shop sees the same address rarely. Standard library only. python3 prices.py
"""

# ================================ CONFIG ====================================
CSV_URL = "https://get.advanced.name/YOUR_LIST_ID.csv"  # CSV export link from your dashboard
COUNTRY = "DE"          # buyers' country: prices are read as local shoppers see them
MIN_SPEED = 3
PRODUCTS_FILE = "products.txt"           # one product URL per line
PRICE_RE = r'"price"\s*:\s*"?([0-9][0-9.,]*)'   # regex, group 1 = the price
OUT_FILE = "prices.csv"                   # url;price;proxy;country is written here
THREADS = 40
TIMEOUT = 20
RETRIES = 3
# ============================================================================

import csv
import io
import itertools
import re
import sys
import threading
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed

COL_IP, COL_PORT, COL_COUNTRY, COL_SPEED = 0, 1, 2, 3
UA = "Mozilla/5.0 (compatible; price-monitor/1.0)"
PRICE = re.compile(PRICE_RE)


def load_proxies():
    raw = urllib.request.urlopen(CSV_URL, timeout=30).read().decode("utf-8", "replace")
    pool = []
    for row in csv.reader(io.StringIO(raw), delimiter=";"):
        if len(row) <= COL_SPEED or not row[COL_PORT].strip().isdigit():
            continue
        if COUNTRY and row[COL_COUNTRY].strip().upper() != COUNTRY.upper():
            continue
        if int(row[COL_SPEED] or 0) < MIN_SPEED:
            continue
        pool.append(row[COL_IP].strip() + ":" + row[COL_PORT].strip())
    return pool


def main():
    proxies = load_proxies()
    if not proxies:
        sys.exit("No proxies for COUNTRY=%r." % COUNTRY)
    with open(PRODUCTS_FILE, encoding="utf-8") as fh:
        products = [line.strip() for line in fh if line.strip() and not line.startswith("#")]
    print("proxies: %d (%s), products: %d" % (len(proxies), COUNTRY, len(products)))

    rotation = itertools.cycle(proxies)
    lock = threading.Lock()

    def grab(url):
        for _ in range(RETRIES):
            with lock:
                proxy = next(rotation)
            handler = urllib.request.ProxyHandler({"http": "http://" + proxy, "https": "http://" + proxy})
            req = urllib.request.Request(url, headers={
                "User-Agent": UA, "Accept-Language": COUNTRY.lower() + "," + COUNTRY.upper()})
            try:
                html = urllib.request.build_opener(handler).open(req, timeout=TIMEOUT).read().decode("utf-8", "replace")
            except Exception:
                continue
            match = PRICE.search(html)
            return (match.group(1) if match else "not found"), proxy
        return "unreachable", ""

    rows = []
    with ThreadPoolExecutor(max_workers=THREADS) as pool:
        futures = {pool.submit(grab, u): u for u in products}
        for fut in as_completed(futures):
            url = futures[fut]
            price, proxy = fut.result()
            rows.append((url, price, proxy, COUNTRY))
            print("%-9s %s" % (price, url[:60]))

    with open(OUT_FILE, "w", encoding="utf-8", newline="") as out:
        writer = csv.writer(out, delimiter=";")
        writer.writerow(["url", "price", "proxy", "country"])
        writer.writerows(rows)
    print("written %s (%d rows)" % (OUT_FILE, len(rows)))


if __name__ == "__main__":
    main()