Configure and set up web scraping environment
{ "environment": { "proxy_enabled": true, "headless_browser": true, "javascript_enabled": true, "cookies_enabled": true } }
{ "default_config": { "timeout": 30000, "retry_attempts": 3, "wait_for_selectors": true, "follow_redirects": true } }
{ "browser": { "type": "chrome", "headless": true, "args": [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage" ], "viewport": { "width": 1920, "height": 1080 } } }
{ "proxies": { "enabled": true, "list": [ { "host": "proxy1.example.com", "port": 8080, "username": "user1", "password": "pass1" } ], "rotation": { "enabled": true, "interval": 100 } } }
{ "rate_limits": { "global": { "requests_per_second": 2 }, "per_domain": { "example.com": { "requests_per_second": 1, "max_concurrent": 2 } } } }
{ "auth": { "type": "basic", "credentials": { "username": "user", "password": "pass" } } }
{ "headers": { "User-Agent": "Custom Scraper 1.0", "Accept": "text/html,application/xhtml+xml", "Accept-Language": "en-US,en;q=0.9", "Connection": "keep-alive" } }
{ "cache": { "enabled": true, "type": "redis", "ttl": 3600, "config": { "host": "localhost", "port": 6379 } } }
{ "error_handling": { "retry_codes": [429, 503], "max_retries": 3, "backoff": { "initial": 1000, "multiplier": 2, "max": 10000 } } }
{ "monitoring": { "enabled": true, "metrics": ["requests_per_minute", "success_rate", "average_response_time"], "alerts": { "error_rate_threshold": 0.1, "response_time_threshold": 5000 } } }