网页采集代理：扛得住封禁潮，大规模采集也不容易被网站封锁

#!/usr/bin/env python3
"""Multithreaded scraper for Advanced.name proxies.

Reads the CSV proxy export, keeps the country and speed you need, then fetches a
list of target URLs through rotating proxies with automatic retries. Standard
library only, no pip. Edit the CONFIG block and run: python3 scrape.py
"""

# ================================ CONFIG ====================================
CSV_URL = "https://get.advanced.name/YOUR_LIST_ID.csv"  # CSV export link from your dashboard
COUNTRY = "DE"          # ISO 3166-1 alpha-2 (e.g. DE, NL, US), or "" for the whole pool
MIN_SPEED = 3           # keep proxies with speed >= this value (higher is better)
THREADS = 50            # parallel requests
TIMEOUT = 15            # seconds per request
RETRIES = 3             # proxies to try before a URL is marked failed
URLS_FILE = "urls.txt"  # one target URL per line
OUT_DIR = "pages"       # fetched HTML is written here as 1.html, 2.html, ...
# ============================================================================

import csv
import io
import itertools
import os
import sys
import threading
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed

# CSV columns from the dashboard export, ';' separated:
# ip ; port ; country ; speed ; uptime ; exit_ip
COL_IP, COL_PORT, COL_COUNTRY, COL_SPEED, COL_UPTIME = 0, 1, 2, 3, 4
UA = "Mozilla/5.0 (compatible; scraper/1.0)"


def load_proxies():
    """Download the CSV list, keep the wanted country and speed, fastest first."""
    raw = urllib.request.urlopen(CSV_URL, timeout=30).read().decode("utf-8", "replace")
    pool = []
    for row in csv.reader(io.StringIO(raw), delimiter=";"):
        if len(row) <= COL_SPEED or not row[COL_PORT].strip().isdigit():
            continue
        country = row[COL_COUNTRY].strip().upper()
        speed = int(row[COL_SPEED] or 0)
        if COUNTRY and country != COUNTRY.upper():
            continue
        if speed < MIN_SPEED:
            continue
        pool.append((row[COL_IP].strip() + ":" + row[COL_PORT].strip(), speed))
    pool.sort(key=lambda item: -item[1])
    return [addr for addr, _ in pool]


def opener_for(proxy):
    handler = urllib.request.ProxyHandler({"http": "http://" + proxy, "https": "http://" + proxy})
    return urllib.request.build_opener(handler)


def main():
    proxies = load_proxies()
    if not proxies:
        sys.exit("No proxies match COUNTRY=%r and MIN_SPEED=%d." % (COUNTRY, MIN_SPEED))
    with open(URLS_FILE, encoding="utf-8") as fh:
        urls = [u for u in map(str.strip, fh) if u and not u.startswith("#")]
    os.makedirs(OUT_DIR, exist_ok=True)
    print("proxies: %d (%s), urls: %d, threads: %d" % (len(proxies), COUNTRY or "any", len(urls), THREADS))

    rotation = itertools.cycle(proxies)
    lock = threading.Lock()
    done = {"ok": 0, "fail": 0}

    def fetch(index, url):
        for _ in range(RETRIES):
            with lock:
                proxy = next(rotation)
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            try:
                body = opener_for(proxy).open(req, timeout=TIMEOUT).read()
            except Exception:
                continue
            with open(os.path.join(OUT_DIR, "%d.html" % index), "wb") as out:
                out.write(body)
            return True
        return False

    with ThreadPoolExecutor(max_workers=THREADS) as pool:
        futures = {pool.submit(fetch, i, u): u for i, u in enumerate(urls, 1)}
        for fut in as_completed(futures):
            ok = fut.result()
            with lock:
                done["ok" if ok else "fail"] += 1
                total = done["ok"] + done["fail"]
            sys.stderr.write("\r%d/%d  ok %d  fail %d" % (total, len(urls), done["ok"], done["fail"]))
            sys.stderr.flush()
    print("\nsaved %d pages to %s/ (%d failed)" % (done["ok"], OUT_DIR, done["fail"]))


if __name__ == "__main__":
    main()