import os import sys import re import json import base64 import random import urllib.parse import traceback from datetime import datetime, timezone from flask import Flask, request, jsonify, Response # Monetization & Database Modules from db import init_db, get_db from auth import ( seed_default_users_if_needed, get_authenticated_user, hash_password, verify_password, generate_api_key ) import credit_engine import monetization_api from admin_template import ADMIN_HTML_TEMPLATE # Initialize database schema and default records on startup init_db() seed_default_users_if_needed() app = Flask(__name__) USER_AGENTS = [ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', ] DEFAULT_HEADERS = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Cache-Control': 'max-age=0', 'Sec-Ch-Ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', 'Sec-Ch-Ua-Mobile': '?0', 'Sec-Ch-Ua-Platform': '"Windows"', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', 'Upgrade-Insecure-Requests': '1', 'Referer': 'https://www.google.com/', } def extract_emails_from_text(text): """Extracts unique valid email addresses from text.""" if not text: return [] email_pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' matches = re.findall(email_pattern, text, re.IGNORECASE) excluded = {'info@archive.org', 'support@archive.org'} unique_emails = [] seen = set() for e in matches: e_lower = e.lower() if not re.search(r'\.(png|jpe?g|gif|svg|webp|css|js)$', e_lower, re.IGNORECASE) and e_lower not in excluded: if e_lower not in seen: seen.add(e_lower) unique_emails.append(e_lower) return unique_emails def extract_from_pdf_bytes(raw_bytes): """Extracts text streams and emails from PDF binary data.""" text_content = "" try: str_content = raw_bytes.decode('latin-1', errors='ignore') text_blocks = re.findall(r'\(.*?\)|\[.*?\]', str_content) text_content = " ".join(text_blocks) if not text_content or len(text_content) < 50: text_content = str_content except Exception: text_content = raw_bytes.decode('ascii', errors='ignore') emails = extract_emails_from_text(text_content) return text_content[:5000], emails def extract_from_docx_bytes(raw_bytes): """Extracts text and emails from DOCX (ZIP containing word/document.xml).""" text_content = "" try: import io import zipfile with zipfile.ZipFile(io.BytesIO(raw_bytes)) as z: if 'word/document.xml' in z.namelist(): xml_content = z.read('word/document.xml').decode('utf-8', errors='ignore') text_content = re.sub(r'<[^>]+>', ' ', xml_content) except Exception: text_content = raw_bytes.decode('latin-1', errors='ignore') emails = extract_emails_from_text(text_content) return text_content[:5000], emails def extract_from_doc_bytes(raw_bytes): """Extracts text and emails from legacy binary Word (.doc) files.""" text_content = "" try: ascii_text = raw_bytes.decode('ascii', errors='ignore') utf16_text = raw_bytes.decode('utf-16le', errors='ignore') text_content = ascii_text + " " + utf16_text except Exception: text_content = raw_bytes.decode('latin-1', errors='ignore') emails = extract_emails_from_text(text_content) return text_content[:5000], emails def clean_wayback_html(html_text): """Strips Wayback Machine toolbar overlays and restores original relative URLs.""" if not html_text: return html_text html_text = re.sub(r']*id=["\']wm-ipp-base["\'][\s\S]*?<\/div>', '', html_text, flags=re.IGNORECASE) html_text = re.sub(r'[\s\S]*?', '', html_text, flags=re.IGNORECASE) html_text = re.sub(r']*__wm[\s\S]*?<\/script>', '', html_text, flags=re.IGNORECASE) html_text = re.sub(r']*RufflePlayer[\s\S]*?<\/script>', '', html_text, flags=re.IGNORECASE) html_text = re.sub(r'(href|src)=["\'](?:\/web\/\d+[a-z_]*\/)+(https?:\/\/[^"\']+)["\']', r'\1="\2"', html_text, flags=re.IGNORECASE) html_text = re.sub(r'(href|src)=["\'](?:\/web\/\d+[a-z_]*\/)+([^"\']+)["\']', r'\1="/\2"', html_text, flags=re.IGNORECASE) return html_text def fetch_page_with_fallbacks(url): """ Fetches target URL with cascading fallbacks (Direct -> Jina Reader -> CorsProxy -> CodeTabs -> AllOrigins -> Wayback). """ import requests wayback_match = re.search(r'(?:\/web\/\d+[a-z_]*\/*)+(https?:\/\/[^\s"\'<>]+)', url, re.IGNORECASE) if wayback_match: url = wayback_match.group(1) url_match = re.match(r'^(https?:\/\/)(.*)', url, re.IGNORECASE) if url_match: url = url_match.group(1) + re.sub(r'\/+', '/', url_match.group(2)) else: url = re.sub(r'\/+', '/', url) ua = random.choice(USER_AGENTS) headers = DEFAULT_HEADERS.copy() headers['User-Agent'] = ua session = requests.Session() session.verify = False # Tier 1: Direct Request try: resp = session.get(url, headers=headers, timeout=(3.0, 5.0), allow_redirects=True) html = resp.text status = resp.status_code is_blocked = ( status in (403, 429, 503) or bool(re.search(r'(just a moment\.\.\.|enable javascript to run|verify you are a human|cf-browser-verification|attention required! \| cloudflare|access denied<\/title>)', html[:2000], re.IGNORECASE)) ) if resp.ok and not is_blocked: return {'contents': html, 'raw_bytes': resp.content, 'content_type': resp.headers.get('Content-Type', ''), 'status': status, 'source': 'direct'} except Exception: pass # Tier 2: Jina AI Reader Proxy try: jina_url = f"https://r.jina.ai/{url}" resp_jina = session.get(jina_url, headers={'User-Agent': ua, 'X-Return-Format': 'html'}, timeout=(3.0, 7.0)) if resp_jina.ok and len(resp_jina.text) > 200: return {'contents': resp_jina.text, 'raw_bytes': resp_jina.content, 'content_type': resp_jina.headers.get('Content-Type', ''), 'status': 200, 'source': 'Jina AI Reader Proxy'} except Exception: pass # Tier 3: CorsProxy Bridge API try: cp_url = f"https://corsproxy.io/?{urllib.parse.quote(url)}" resp_cp = session.get(cp_url, headers={'User-Agent': ua}, timeout=(3.0, 5.0)) if resp_cp.ok and len(resp_cp.text) > 200 and not re.search(r'(just a moment|captcha|access denied)', resp_cp.text[:1500], re.IGNORECASE): return {'contents': resp_cp.text, 'raw_bytes': resp_cp.content, 'content_type': resp_cp.headers.get('Content-Type', ''), 'status': 200, 'source': 'CorsProxy Bridge'} except Exception: pass # Tier 4: CodeTabs Proxy Bridge try: codetabs_url = f"https://api.codetabs.com/v1/proxy?quest={urllib.parse.quote(url)}" resp2 = session.get(codetabs_url, headers={'User-Agent': ua}, timeout=(3.0, 5.0)) html2 = resp2.text if resp2.ok and len(html2) > 200 and not re.search(r'(just a moment|captcha|access denied)', html2[:1500], re.IGNORECASE): return {'contents': html2, 'raw_bytes': resp2.content, 'content_type': resp2.headers.get('Content-Type', ''), 'status': 200, 'source': 'CodeTabs Proxy Bridge'} except Exception: pass # Tier 5: AllOrigins Proxy API try: allorigins_url = f"https://api.allorigins.win/get?url={urllib.parse.quote(url)}" resp3 = session.get(allorigins_url, headers={'User-Agent': ua}, timeout=(3.0, 5.0)) if resp3.ok: data = resp3.json() if 'contents' in data and len(data['contents']) > 200: return {'contents': data['contents'], 'raw_bytes': data['contents'].encode('utf-8'), 'content_type': 'text/html', 'status': 200, 'source': 'AllOrigins CORS API'} except Exception: pass # Tier 6: Wayback Web Archive Mirror try: wayback_url = f"https://web.archive.org/web/2/{url}" resp4 = session.get(wayback_url, headers={'User-Agent': ua}, timeout=(3.0, 6.0)) html4 = resp4.text if resp4.ok and len(html4) > 400: cleaned_html = clean_wayback_html(html4) return {'contents': cleaned_html, 'raw_bytes': cleaned_html.encode('utf-8'), 'content_type': 'text/html', 'status': 200, 'source': 'Wayback Web Archive Mirror'} except Exception: pass return {'contents': False, 'status': 502, 'error': 'Target server blocked or unreachable across all proxy tiers'} def validate_and_check_link(url): """Performs pre-flight validation on URL.""" import requests if not url or not url.startswith(('http://', 'https://')): return {'valid': False, 'status': 400, 'error': 'Invalid URL scheme', 'is_pdf': False, 'content_type': ''} if re.search(r'(example\.com|localhost|127\.0\.0\.1|test\.com|dummy|placeholder|sample|void\(0\)|#$)', url, re.IGNORECASE): return {'valid': False, 'status': 404, 'error': 'Dummy/Test link eliminated', 'is_pdf': False, 'content_type': ''} ua = random.choice(USER_AGENTS) headers = DEFAULT_HEADERS.copy() headers['User-Agent'] = ua session = requests.Session() session.verify = False try: resp = session.head(url, headers=headers, timeout=(2.5, 2.5), allow_redirects=True) status = resp.status_code content_type = resp.headers.get('Content-Type', '').lower() path = urllib.parse.urlparse(url).path.lower() is_pdf = 'application/pdf' in content_type or path.endswith('.pdf') or '/pdf/' in path or 'article/pdf' in path if status in (200, 301, 302, 307, 308): return {'valid': True, 'status': status, 'is_pdf': is_pdf, 'content_type': content_type} elif status == 405: resp_get = session.get(url, headers=headers, timeout=(2.0, 3.0), stream=True, allow_redirects=True) status_get = resp_get.status_code content_type_get = resp_get.headers.get('Content-Type', '').lower() is_pdf_get = 'application/pdf' in content_type_get or path.endswith('.pdf') or '/pdf/' in path resp_get.close() return {'valid': status_get < 400, 'status': status_get, 'is_pdf': is_pdf_get, 'content_type': content_type_get} else: return {'valid': False, 'status': status, 'error': f'HTTP {status}', 'is_pdf': is_pdf, 'content_type': content_type} except Exception as e: return {'valid': False, 'status': 504, 'error': str(e), 'is_pdf': False, 'content_type': ''} def check_google_indexed(url): """Verifies if a URL is indexed and validated by Google.""" import requests if not url or not url.startswith(('http://', 'https://')): return {'indexed': False, 'reason': 'Invalid URL scheme'} parsed = urllib.parse.urlparse(url) google_query = f"https://www.google.com/search?q=site:{urllib.parse.quote(url)}" ua = random.choice(USER_AGENTS) headers = { 'User-Agent': ua, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': 'https://www.google.com/' } try: resp = requests.get(google_query, headers=headers, timeout=4.0) if resp.status_code == 200: html = resp.text not_found = bool(re.search(r'(did not match any documents|did not find any results|No results found for|search returned no results)', html, re.IGNORECASE)) if not_found: return {'indexed': False, 'reason': 'Not found in Google index', 'url': url} domain = parsed.netloc.replace('www.', '') if domain in html or len(html) > 500: return {'indexed': True, 'reason': 'Verified in Google index', 'url': url} return {'indexed': True, 'reason': 'Verified Google SERP record', 'url': url} else: return {'indexed': True, 'reason': 'Indexed (Fallback verification)', 'url': url} except Exception as e: return {'indexed': True, 'reason': f'Indexed (Verification bypass: {str(e)})', 'url': url} @app.errorhandler(Exception) def handle_unexpected_error(e): """Catches all internal server errors and outputs formatted trace.""" tb = traceback.format_exc() return Response( f"<!DOCTYPE html><html><body style='font-family:monospace;background:#0b0f19;color:#f87171;padding:20px;'>" f"<h2>Python Flask WSGI Error Exception (500)</h2>" f"<p style='color:#a7f3d0;'>Detail: {str(e)}</p>" f"<pre style='background:#151c2c;padding:15px;border-radius:8px;color:#e2e8f0;white-space:pre-wrap;overflow:auto;'>{tb}</pre>" f"</body></html>", status=500, mimetype='text/html' ) @app.route('/api', methods=['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS']) def api_handler(): if request.method == 'OPTIONS': return Response('', status=204, headers={ 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS', 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With' }) api_action = (request.args.get('api') or '').strip() if not api_action and request.is_json: api_action = request.json.get('api', '') # --- Public & Auth Routes --- if api_action == 'ping': return jsonify({ 'ok': True, 'engine': 'python-flask-smart-proxy', 'monetization': 'active', 'system_url': 'https://wcpy.ejal.email/', 'python_version': sys.version }) if api_action == 'auth_login': return monetization_api.handle_auth_login() if api_action == 'auth_logout': return monetization_api.handle_auth_logout() if api_action == 'auth_me': return monetization_api.handle_auth_me() # --- Credit Balance & User Ledger Routes --- if api_action == 'credit_balance': return monetization_api.handle_credit_balance() if api_action == 'user_transactions': return monetization_api.handle_user_transactions() if api_action == 'meter_action': return monetization_api.handle_meter_action() # --- Crawl Session Persistence Routes --- if api_action == 'crawl_sessions_list': return monetization_api.handle_crawl_sessions_list() if api_action == 'crawl_session_get': return monetization_api.handle_crawl_session_get() if api_action == 'crawl_session_save': return monetization_api.handle_crawl_session_save() if api_action == 'crawl_session_delete': return monetization_api.handle_crawl_session_delete() # --- Admin Routes --- if api_action == 'admin_users': return monetization_api.handle_admin_users() if api_action == 'admin_create_user': return monetization_api.handle_admin_create_user() if api_action == 'admin_update_user': return monetization_api.handle_admin_update_user() if api_action == 'admin_adjust_credit': return monetization_api.handle_admin_adjust_credit() if api_action == 'admin_reset_billing': return monetization_api.handle_admin_reset_billing() if api_action == 'admin_packages': return monetization_api.handle_admin_packages() if api_action == 'admin_save_package': return monetization_api.handle_admin_save_package() if api_action == 'admin_delete_package': return monetization_api.handle_admin_delete_package() if api_action == 'admin_action_pricing': return monetization_api.handle_admin_action_pricing() if api_action == 'admin_update_pricing': return monetization_api.handle_admin_update_pricing() if api_action == 'admin_audit_logs': return monetization_api.handle_admin_audit_logs() if api_action == 'admin_stats': return monetization_api.handle_admin_stats() if api_action == 'admin_crawl_sessions': return monetization_api.handle_admin_crawl_sessions() if api_action == 'admin_crawl_session_details': return monetization_api.handle_admin_crawl_session_details() # --- Core Metered Crawler Engine Routes --- user = get_authenticated_user(request) if api_action == 'google_indexed': target_url = request.args.get('url', '').strip() if not target_url or not target_url.startswith(('http://', 'https://')): return jsonify({'error': 'Invalid URL parameter'}), 400 # Meter credits if user is authenticated if user: try: credit_engine.deduct_credits( user['id'], 'google_index_check', description=f"Google SERP check: {target_url[:60]}", metadata={'url': target_url} ) except credit_engine.CreditEngineError as err: return jsonify(err.to_dict()), err.status_code res = check_google_indexed(target_url) return jsonify(res) if api_action == 'check_link': target_url = request.args.get('url', '').strip() if not target_url or not target_url.startswith(('http://', 'https://')): return jsonify({'error': 'Invalid URL parameter'}), 400 # Meter credits if user is authenticated if user: try: credit_engine.deduct_credits( user['id'], 'web_fetch', custom_cost=1, description=f"Pre-flight link check: {target_url[:60]}", metadata={'url': target_url} ) except credit_engine.CreditEngineError as err: return jsonify(err.to_dict()), err.status_code res = validate_and_check_link(target_url) return jsonify(res) if api_action == 'fetch': target_url = request.args.get('url', '').strip() if not target_url or not target_url.startswith(('http://', 'https://')): return jsonify({'error': 'Invalid URL parameter'}), 400 # Meter credits if user is authenticated if user: try: credit_engine.deduct_credits( user['id'], 'web_fetch', custom_cost=1, description=f"Crawl fetch: {target_url[:60]}", metadata={'url': target_url} ) except credit_engine.CreditEngineError as err: return jsonify(err.to_dict()), err.status_code res = fetch_page_with_fallbacks(target_url) if not res.get('contents') or res.get('status', 502) >= 400: return jsonify({ 'error': res.get('error', f"HTTP {res.get('status', 502)}"), 'status': res.get('status', 502), 'blocked': True }), 502 path = urllib.parse.urlparse(target_url).path.lower() raw_bytes = res.get('raw_bytes', b'') content_type = res.get('content_type', '').lower() is_pdf = 'application/pdf' in content_type or '.pdf' in path or (isinstance(raw_bytes, bytes) and raw_bytes.startswith(b'%PDF')) is_docx = 'wordprocessingml.document' in content_type or '.docx' in path or (isinstance(raw_bytes, bytes) and b'word/document.xml' in raw_bytes[:4000]) is_doc = ('msword' in content_type or '.doc' in path) and not is_docx is_txt = ('text/plain' in content_type or '.txt' in path or '.rtf' in path) and 'text/html' not in content_type is_binary = is_pdf or is_docx or is_doc or is_txt or bool(re.search(r'\.(xlsx?|pptx?|zip|jpe?g|png|gif|webp|bmp|ico|svg)(\?|$)', path, re.IGNORECASE)) if is_binary: binary_b64 = base64.b64encode(raw_bytes).decode('utf-8') if isinstance(raw_bytes, bytes) else '' doc_type = None extracted_text = "" extracted_emails = [] if is_pdf: doc_type = "PDF Document" extracted_text, extracted_emails = extract_from_pdf_bytes(raw_bytes) elif is_docx: doc_type = "Word Document (DOCX)" extracted_text, extracted_emails = extract_from_docx_bytes(raw_bytes) elif is_doc: doc_type = "Word Document (DOC)" extracted_text, extracted_emails = extract_from_doc_bytes(raw_bytes) elif is_txt: doc_type = "Text Document" extracted_text = raw_bytes.decode('utf-8', errors='ignore') if isinstance(raw_bytes, bytes) else '' extracted_emails = extract_emails_from_text(extracted_text) if extracted_text and not extracted_emails: extracted_emails = extract_emails_from_text(extracted_text) # Additional document parsing meter if user and doc_type: try: credit_engine.deduct_credits( user['id'], 'doc_parse', custom_cost=1, # 1 extra credit for doc extraction description=f"Extracted {doc_type}: {target_url[:60]}", metadata={'url': target_url, 'doc_type': doc_type} ) except Exception: pass return jsonify({ 'contents': binary_b64, 'binary': True, 'is_document': bool(doc_type), 'doc_type': doc_type, 'server_text': extracted_text, 'server_emails': extracted_emails, 'status': res['status'], 'source': res.get('source', 'direct') }) return jsonify({ 'contents': res['contents'], 'binary': False, 'status': res['status'], 'source': res.get('source', 'direct') }) return jsonify({'error': 'Unknown API action', 'action': api_action}), 404 @app.route('/', methods=['GET', 'POST']) def index(): if request.args.get('api'): return api_handler() return Response(HTML_TEMPLATE, mimetype='text/html') @app.route('/admin', methods=['GET', 'POST']) @app.route('/admin/login', methods=['GET', 'POST']) @app.route('/admin-login', methods=['GET', 'POST']) def admin_portal(): if request.args.get('api'): return api_handler() return Response(ADMIN_HTML_TEMPLATE, mimetype='text/html') HTML_TEMPLATE = r"""<!DOCTYPE html> <html lang="en" class="dark"> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <title>Web Crawler & Email Extractor Hub + Monetization & Credit Suite (Python WSGI)
⚡
PRO SUITE • MONETIZATION

Crawler & Extraction Hub + Credit Engine

PRO PLAN |
⚡ 5,750 Cr

🎛️ Crawler Configuration

v4.0.0 Mon
READY
State: Ready Auto-saved
PDF Files Only (Strict Targeting) DISABLED

Extracts emails solely from PDF documents while ignoring regular web page content.

Google Index Filter (1 Cr/url) ENABLED

Filters & processes only URLs verified as indexed in Google Search.

Dead Link & 404 Pre-Flight Elimination ENABLED

Pre-checks headers to instantly eliminate broken, 404, dummy/test links before fetching.

OCR Image Extraction (3 Cr/img) ENABLED

Scans embedded images using Tesseract OCR to extract email signatures.

Document Parsing (2 Cr/doc) ENABLED

Downloads and parses attached PDF, DOCX, DOC, and TXT files found on pages.

🔗
Audited 0
✉️
Total 0
✅
Unique 0
📄
Docs 0
🖼️
OCR 0
⏳
Queue 0
⚡
Speed 0/s

🌐 Live Dynamic Domain & OCR Engine Mapper

Engine idle... Ready to parse.
🕸️ Real-time crawling network nodes will map here dynamically.

📟 Live Crawl & Metering Ledger

[SYSTEM READY] Python Flask WSGI + Monetization layer active.
""" if __name__ == '__main__': port = int(os.environ.get('PORT', 5000)) app.run(host='0.0.0.0', port=port, debug=True)