# 網(wǎng)頁請求和解析
pip install requests
pip install beautifulsoup4
pip install selenium
pip install lxml

# 瀏覽器自動化（用于JavaScript渲染的頁面）
pip install webdriver-manager

# 數(shù)據(jù)處理和存儲
pip install pandas
pip install sqlalchemy

# 通知服務(wù)
pip install smtplib
pip install requests # 用于webhook

# 任務(wù)調(diào)度
pip install schedule
pip install APScheduler

# 其他工具
pip install hashlib
pip install difflib
pip install pillow # 用于截圖比較

# 導(dǎo)入所需庫
import requests
from bs4 import BeautifulSoup
import hashlib
import time
import smtplib
import schedule
import json
import sqlite3
from datetime import datetime
import logging
from typing import Dict, List, Optional, Tuple
import difflib
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
import os
from pathlib import Path

class WebsiteMonitor:
    """
    網(wǎng)站變化監(jiān)控器 - 核心監(jiān)控引擎
    提供網(wǎng)頁內(nèi)容監(jiān)控、變化檢測和通知功能
    """
    
    def __init__(self, config_file: str = "monitor_config.json"):
        """
        初始化網(wǎng)站監(jiān)控器
        
        Args:
            config_file: 配置文件路徑
        """
        self.config_file = Path(config_file)
        self.config = self.load_config()
        self.setup_logging()
        self.setup_database()
        
    def setup_logging(self):
        """設(shè)置日志記錄"""
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler('website_monitor.log', encoding='utf-8'),
                logging.StreamHandler()
            ]
        )
        self.logger = logging.getLogger(__name__)
    
    def load_config(self) -> Dict:
        """
        加載監(jiān)控配置
        
        Returns:
            Dict: 配置信息
        """
        default_config = {
            "monitoring_interval": 300,  # 默認(rèn)5分鐘
            "notification_methods": {
                "email": {
                    "enabled": False,
                    "smtp_server": "smtp.gmail.com",
                    "smtp_port": 587,
                    "username": "your_email@gmail.com",
                    "password": "your_password",
                    "recipient": "recipient@example.com"
                },
                "webhook": {
                    "enabled": False,
                    "url": "https://your-webhook-url.com"
                }
            },
            "websites": {},
            "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
        }
        
        try:
            if self.config_file.exists():
                with open(self.config_file, 'r', encoding='utf-8') as f:
                    loaded_config = json.load(f)
                    # 合并配置，確保包含所有必要字段
                    return self.merge_configs(default_config, loaded_config)
            else:
                # 創(chuàng)建默認(rèn)配置文件
                self.save_config(default_config)
                return default_config
        except Exception as e:
            self.logger.error(f"加載配置文件失敗: {str(e)}")
            return default_config
    
    def merge_configs(self, default: Dict, user: Dict) -> Dict:
        """
        合并默認(rèn)配置和用戶配置
        
        Args:
            default: 默認(rèn)配置
            user: 用戶配置
            
        Returns:
            Dict: 合并后的配置
        """
        result = default.copy()
        
        def deep_merge(default_dict, user_dict):
            for key, value in user_dict.items():
                if key in default_dict and isinstance(default_dict[key], dict) and isinstance(value, dict):
                    deep_merge(default_dict[key], value)
                else:
                    default_dict[key] = value
        
        deep_merge(result, user)
        return result
    
    def save_config(self, config: Dict = None):
        """
        保存配置到文件
        
        Args:
            config: 要保存的配置，如果為None則保存當(dāng)前配置
        """
        if config is None:
            config = self.config
        
        try:
            with open(self.config_file, 'w', encoding='utf-8') as f:
                json.dump(config, f, indent=2, ensure_ascii=False)
            self.logger.info("配置已保存")
        except Exception as e:
            self.logger.error(f"保存配置失敗: {str(e)}")
    
    def setup_database(self):
        """設(shè)置SQLite數(shù)據(jù)庫"""
        try:
            self.db_path = Path("website_monitor.db")
            self.conn = sqlite3.connect(self.db_path)
            self.create_tables()
            self.logger.info("數(shù)據(jù)庫初始化完成")
        except Exception as e:
            self.logger.error(f"數(shù)據(jù)庫初始化失敗: {str(e)}")
    
    def create_tables(self):
        """創(chuàng)建數(shù)據(jù)庫表"""
        cursor = self.conn.cursor()
        
        # 創(chuàng)建網(wǎng)站監(jiān)控記錄表
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS website_changes (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                website_url TEXT NOT NULL,
                change_type TEXT NOT NULL,
                change_description TEXT,
                previous_content_hash TEXT,
                current_content_hash TEXT,
                change_timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
                notified BOOLEAN DEFAULT 0
            )
        ''')
        
        # 創(chuàng)建內(nèi)容快照表
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS content_snapshots (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                website_url TEXT NOT NULL,
                content_hash TEXT NOT NULL,
                content_text TEXT,
                snapshot_timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
            )
        ''')
        
        # 創(chuàng)建監(jiān)控配置表
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS monitor_configs (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                website_url TEXT UNIQUE NOT NULL,
                check_interval INTEGER DEFAULT 300,
                css_selector TEXT,
                enabled BOOLEAN DEFAULT 1,
                last_checked DATETIME
            )
        ''')
        
        self.conn.commit()
    
    def add_website(self, url: str, check_interval: int = 300, 
                   css_selector: str = None, enabled: bool = True) -> bool:
        """
        添加要監(jiān)控的網(wǎng)站
        
        Args:
            url: 網(wǎng)站URL
            check_interval: 檢查間隔（秒）
            css_selector: CSS選擇器（用于監(jiān)控特定部分）
            enabled: 是否啟用監(jiān)控
            
        Returns:
            bool: 添加是否成功
        """
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                INSERT OR REPLACE INTO monitor_configs 
                (website_url, check_interval, css_selector, enabled, last_checked)
                VALUES (?, ?, ?, ?, ?)
            ''', (url, check_interval, css_selector, enabled, None))
            
            self.conn.commit()
            
            # 更新內(nèi)存中的配置
            if 'websites' not in self.config:
                self.config['websites'] = {}
            
            self.config['websites'][url] = {
                'check_interval': check_interval,
                'css_selector': css_selector,
                'enabled': enabled
            }
            
            self.save_config()
            self.logger.info(f"已添加監(jiān)控網(wǎng)站: {url}")
            return True
            
        except Exception as e:
            self.logger.error(f"添加網(wǎng)站失敗 {url}: {str(e)}")
            return False
    
    def remove_website(self, url: str) -> bool:
        """
        移除監(jiān)控網(wǎng)站
        
        Args:
            url: 網(wǎng)站URL
            
        Returns:
            bool: 移除是否成功
        """
        try:
            cursor = self.conn.cursor()
            cursor.execute('DELETE FROM monitor_configs WHERE website_url = ?', (url,))
            self.conn.commit()
            
            # 更新內(nèi)存中的配置
            if url in self.config.get('websites', {}):
                del self.config['websites'][url]
                self.save_config()
            
            self.logger.info(f"已移除監(jiān)控網(wǎng)站: {url}")
            return True
            
        except Exception as e:
            self.logger.error(f"移除網(wǎng)站失敗 {url}: {str(e)}")
            return False
    
    def fetch_web_content(self, url: str, css_selector: str = None) -> Tuple[Optional[str], Optional[str]]:
        """
        獲取網(wǎng)頁內(nèi)容
        
        Args:
            url: 網(wǎng)頁URL
            css_selector: CSS選擇器（用于提取特定部分）
            
        Returns:
            Tuple: (原始內(nèi)容, 提取的內(nèi)容)
        """
        try:
            headers = {
                'User-Agent': self.config.get('user_agent', 
                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
            }
            
            response = requests.get(url, headers=headers, timeout=30)
            response.raise_for_status()
            
            raw_content = response.text
            
            # 如果指定了CSS選擇器，提取特定部分
            extracted_content = None
            if css_selector:
                soup = BeautifulSoup(raw_content, 'html.parser')
                selected_elements = soup.select(css_selector)
                if selected_elements:
                    extracted_content = '\n'.join([elem.get_text(strip=True) for elem in selected_elements])
            
            self.logger.debug(f"成功獲取網(wǎng)頁內(nèi)容: {url}")
            return raw_content, extracted_content
            
        except requests.RequestException as e:
            self.logger.error(f"獲取網(wǎng)頁內(nèi)容失敗 {url}: {str(e)}")
            return None, None
        except Exception as e:
            self.logger.error(f"解析網(wǎng)頁內(nèi)容失敗 {url}: {str(e)}")
            return None, None
    
    def calculate_content_hash(self, content: str) -> str:
        """
        計(jì)算內(nèi)容哈希值
        
        Args:
            content: 文本內(nèi)容
            
        Returns:
            str: 內(nèi)容的MD5哈希值
        """
        return hashlib.md5(content.encode('utf-8')).hexdigest()
    
    def save_content_snapshot(self, url: str, content_hash: str, content_text: str = None):
        """
        保存內(nèi)容快照到數(shù)據(jù)庫
        
        Args:
            url: 網(wǎng)站URL
            content_hash: 內(nèi)容哈希值
            content_text: 內(nèi)容文本（可選）
        """
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                INSERT INTO content_snapshots (website_url, content_hash, content_text)
                VALUES (?, ?, ?)
            ''', (url, content_hash, content_text))
            self.conn.commit()
        except Exception as e:
            self.logger.error(f"保存內(nèi)容快照失敗: {str(e)}")
    
    def get_previous_content_hash(self, url: str) -> Optional[str]:
        """
        獲取上一次的內(nèi)容哈希值
        
        Args:
            url: 網(wǎng)站URL
            
        Returns:
            str: 上一次的內(nèi)容哈希值，如果沒有記錄則返回None
        """
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                SELECT content_hash FROM content_snapshots 
                WHERE website_url = ? 
                ORDER BY snapshot_timestamp DESC 
                LIMIT 1
            ''', (url,))
            
            result = cursor.fetchone()
            return result[0] if result else None
            
        except Exception as e:
            self.logger.error(f"獲取歷史哈希值失敗 {url}: {str(e)}")
            return None
    
    def detect_changes(self, url: str, current_content: str, 
                      previous_hash: str = None) -> Tuple[bool, Optional[str], Optional[str]]:
        """
        檢測內(nèi)容變化
        
        Args:
            url: 網(wǎng)站URL
            current_content: 當(dāng)前內(nèi)容
            previous_hash: 上一次的內(nèi)容哈希值
            
        Returns:
            Tuple: (是否變化, 當(dāng)前哈希值, 變化描述)
        """
        if not current_content:
            return False, None, "無法獲取當(dāng)前內(nèi)容"
        
        current_hash = self.calculate_content_hash(current_content)
        
        # 如果沒有歷史記錄，保存初始快照并返回?zé)o變化
        if previous_hash is None:
            self.save_content_snapshot(url, current_hash, current_content)
            return False, current_hash, "首次監(jiān)控，建立基準(zhǔn)"
        
        # 比較哈希值
        if current_hash == previous_hash:
            return False, current_hash, "內(nèi)容未變化"
        
        # 檢測到變化，獲取詳細(xì)差異
        previous_content = self.get_previous_content_text(url)
        change_description = self.analyze_content_changes(previous_content, current_content)
        
        # 保存新快照
        self.save_content_snapshot(url, current_hash, current_content)
        
        return True, current_hash, change_description
    
    def get_previous_content_text(self, url: str) -> Optional[str]:
        """
        獲取上一次的內(nèi)容文本
        
        Args:
            url: 網(wǎng)站URL
            
        Returns:
            str: 上一次的內(nèi)容文本
        """
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                SELECT content_text FROM content_snapshots 
                WHERE website_url = ? 
                ORDER BY snapshot_timestamp DESC 
                LIMIT 1
            ''', (url,))
            
            result = cursor.fetchone()
            return result[0] if result else None
            
        except Exception as e:
            self.logger.error(f"獲取歷史內(nèi)容失敗 {url}: {str(e)}")
            return None
    
    def analyze_content_changes(self, old_content: str, new_content: str) -> str:
        """
        分析內(nèi)容變化詳情
        
        Args:
            old_content: 舊內(nèi)容
            new_content: 新內(nèi)容
            
        Returns:
            str: 變化描述
        """
        if not old_content or not new_content:
            return "無法比較內(nèi)容變化"
        
        # 使用difflib進(jìn)行文本比較
        diff = difflib.unified_diff(
            old_content.splitlines(keepends=True),
            new_content.splitlines(keepends=True),
            fromfile='舊內(nèi)容',
            tofile='新內(nèi)容',
            n=3
        )
        
        diff_text = ''.join(diff)
        
        if diff_text:
            # 簡化diff輸出，只顯示前幾行
            lines = diff_text.split('\n')[:10]
            return "檢測到內(nèi)容變化:\n" + '\n'.join(lines)
        else:
            return "內(nèi)容發(fā)生變化但無法生成差異報(bào)告"
    
    def check_website(self, url: str) -> Dict:
        """
        檢查單個網(wǎng)站的變化
        
        Args:
            url: 網(wǎng)站URL
            
        Returns:
            Dict: 檢查結(jié)果
        """
        self.logger.info(f"檢查網(wǎng)站: {url}")
        
        # 獲取網(wǎng)站配置
        website_config = self.config['websites'].get(url, {})
        css_selector = website_config.get('css_selector')
        
        # 獲取網(wǎng)頁內(nèi)容
        raw_content, extracted_content = self.fetch_web_content(url, css_selector)
        
        # 使用提取的內(nèi)容（如果有），否則使用原始內(nèi)容
        content_to_check = extracted_content if extracted_content else raw_content
        
        if not content_to_check:
            return {
                'url': url,
                'changed': False,
                'error': '無法獲取網(wǎng)頁內(nèi)容',
                'timestamp': datetime.now()
            }
        
        # 獲取上一次的內(nèi)容哈希
        previous_hash = self.get_previous_content_hash(url)
        
        # 檢測變化
        changed, current_hash, change_description = self.detect_changes(
            url, content_to_check, previous_hash
        )
        
        result = {
            'url': url,
            'changed': changed,
            'current_hash': current_hash,
            'previous_hash': previous_hash,
            'change_description': change_description,
            'timestamp': datetime.now()
        }
        
        if changed:
            self.logger.info(f"檢測到變化: {url}")
            # 記錄變化到數(shù)據(jù)庫
            self.record_change(url, change_description, previous_hash, current_hash)
            # 發(fā)送通知
            self.send_notification(url, change_description)
        
        return result
    
    def record_change(self, url: str, change_description: str, 
                     previous_hash: str, current_hash: str):
        """
        記錄變化到數(shù)據(jù)庫
        
        Args:
            url: 網(wǎng)站URL
            change_description: 變化描述
            previous_hash: 之前的哈希值
            current_hash: 當(dāng)前哈希值
        """
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                INSERT INTO website_changes 
                (website_url, change_type, change_description, previous_content_hash, current_content_hash)
                VALUES (?, ?, ?, ?, ?)
            ''', (url, 'content_change', change_description, previous_hash, current_hash))
            self.conn.commit()
        except Exception as e:
            self.logger.error(f"記錄變化失敗: {str(e)}")
    
    def send_notification(self, url: str, change_description: str):
        """
        發(fā)送變化通知
        
        Args:
            url: 網(wǎng)站URL
            change_description: 變化描述
        """
        notification_methods = self.config.get('notification_methods', {})
        
        # 郵件通知
        if notification_methods.get('email', {}).get('enabled', False):
            self.send_email_notification(url, change_description)
        
        # Webhook通知
        if notification_methods.get('webhook', {}).get('enabled', False):
            self.send_webhook_notification(url, change_description)
    
    def send_email_notification(self, url: str, change_description: str):
        """
        發(fā)送郵件通知
        
        Args:
            url: 網(wǎng)站URL
            change_description: 變化描述
        """
        try:
            email_config = self.config['notification_methods']['email']
            
            subject = f"網(wǎng)站變化通知: {url}"
            body = f"""
檢測到網(wǎng)站內(nèi)容發(fā)生變化:

網(wǎng)站: {url}
時間: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

變化詳情:
{change_description}

---
此通知由網(wǎng)站監(jiān)控系統(tǒng)自動發(fā)送
            """
            
            # 發(fā)送郵件（這里需要實(shí)現(xiàn)具體的郵件發(fā)送邏輯）
            self._send_email(
                email_config['smtp_server'],
                email_config['smtp_port'],
                email_config['username'],
                email_config['password'],
                email_config['recipient'],
                subject,
                body
            )
            
            self.logger.info(f"已發(fā)送郵件通知: {url}")
            
        except Exception as e:
            self.logger.error(f"發(fā)送郵件通知失敗: {str(e)}")
    
    def _send_email(self, smtp_server: str, port: int, username: str, 
                   password: str, recipient: str, subject: str, body: str):
        """
        發(fā)送郵件（具體實(shí)現(xiàn)）
        
        Args:
            smtp_server: SMTP服務(wù)器
            port: 端口
            username: 發(fā)件人郵箱
            password: 密碼
            recipient: 收件人郵箱
            subject: 郵件主題
            body: 郵件正文
        """
        # 這里實(shí)現(xiàn)具體的郵件發(fā)送邏輯
        # 可以使用smtplib和email庫
        pass
    
    def send_webhook_notification(self, url: str, change_description: str):
        """
        發(fā)送Webhook通知
        
        Args:
            url: 網(wǎng)站URL
            change_description: 變化描述
        """
        try:
            webhook_config = self.config['notification_methods']['webhook']
            webhook_url = webhook_config['url']
            
            payload = {
                'url': url,
                'change_description': change_description,
                'timestamp': datetime.now().isoformat(),
                'type': 'website_change'
            }
            
            response = requests.post(webhook_url, json=payload, timeout=10)
            response.raise_for_status()
            
            self.logger.info(f"已發(fā)送Webhook通知: {url}")
            
        except Exception as e:
            self.logger.error(f"發(fā)送Webhook通知失敗: {str(e)}")
    
    def check_all_websites(self) -> List[Dict]:
        """
        檢查所有監(jiān)控的網(wǎng)站
        
        Returns:
            List: 所有網(wǎng)站的檢查結(jié)果
        """
        results = []
        websites = self.config.get('websites', {})
        
        for url, config in websites.items():
            if config.get('enabled', True):
                try:
                    result = self.check_website(url)
                    results.append(result)
                    
                    # 避免請求過于頻繁
                    time.sleep(1)
                    
                except Exception as e:
                    self.logger.error(f"檢查網(wǎng)站失敗 {url}: {str(e)}")
                    results.append({
                        'url': url,
                        'changed': False,
                        'error': str(e),
                        'timestamp': datetime.now()
                    })
        
        return results
    
    def start_monitoring(self):
        """開始定時監(jiān)控"""
        self.logger.info("啟動網(wǎng)站監(jiān)控服務(wù)")
        
        interval = self.config.get('monitoring_interval', 300)
        
        # 使用schedule庫設(shè)置定時任務(wù)
        schedule.every(interval).seconds.do(self.check_all_websites)
        
        # 立即執(zhí)行一次檢查
        self.check_all_websites()
        
        # 保持程序運(yùn)行
        try:
            while True:
                schedule.run_pending()
                time.sleep(1)
        except KeyboardInterrupt:
            self.logger.info("監(jiān)控服務(wù)已停止")
        finally:
            self.cleanup()
    
    def cleanup(self):
        """清理資源"""
        if hasattr(self, 'conn'):
            self.conn.close()
        self.logger.info("資源清理完成")

# 使用示例
def demo_basic_monitor():
    """演示基礎(chǔ)監(jiān)控功能"""
    monitor = WebsiteMonitor()
    
    # 添加要監(jiān)控的網(wǎng)站
    print("=== 添加監(jiān)控網(wǎng)站 ===")
    monitor.add_website(
        "https://httpbin.org/json",
        check_interval=300,  # 5分鐘
        css_selector=None    # 監(jiān)控整個頁面
    )
    
    monitor.add_website(
        "https://example.com",
        check_interval=600,  # 10分鐘
        css_selector="h1"    # 只監(jiān)控h1標(biāo)簽
    )
    
    # 執(zhí)行一次檢查
    print("\n=== 執(zhí)行網(wǎng)站檢查 ===")
    results = monitor.check_all_websites()
    
    for result in results:
        status = "變化" if result['changed'] else "無變化"
        print(f"網(wǎng)站: {result['url']} - {status}")
        if result.get('error'):
            print(f"  錯誤: {result['error']}")
        if result.get('change_description'):
            print(f"  變化: {result['change_description'][:100]}...")
    
    # 顯示監(jiān)控統(tǒng)計(jì)
    print("\n=== 監(jiān)控統(tǒng)計(jì) ===")
    print(f"監(jiān)控網(wǎng)站數(shù)量: {len(monitor.config.get('websites', {}))}")
    
    return monitor

if __name__ == "__main__":
    demo_basic_monitor()

class AdvancedWebsiteMonitor(WebsiteMonitor):
    """
    高級網(wǎng)站監(jiān)控器 - 支持JavaScript渲染和更復(fù)雜的監(jiān)控策略
    """
    
    def __init__(self, config_file: str = "monitor_config.json"):
        """初始化高級監(jiān)控器"""
        super().__init__(config_file)
        self.driver = None
    
    def setup_selenium_driver(self, headless: bool = True):
        """
        設(shè)置Selenium WebDriver
        
        Args:
            headless: 是否使用無頭模式
        """
        try:
            chrome_options = Options()
            if headless:
                chrome_options.add_argument("--headless")
            chrome_options.add_argument("--no-sandbox")
            chrome_options.add_argument("--disable-dev-shm-usage")
            chrome_options.add_argument("--disable-gpu")
            chrome_options.add_argument("--window-size=1920,1080")
            
            self.driver = webdriver.Chrome(
                service=Service(ChromeDriverManager().install()),
                options=chrome_options
            )
            self.logger.info("Selenium WebDriver初始化完成")
        except Exception as e:
            self.logger.error(f"Selenium WebDriver初始化失敗: {str(e)}")
    
    def fetch_web_content_selenium(self, url: str, css_selector: str = None, 
                                 wait_time: int = 5) -> Tuple[Optional[str], Optional[str]]:
        """
        使用Selenium獲取網(wǎng)頁內(nèi)容（支持JavaScript）
        
        Args:
            url: 網(wǎng)頁URL
            css_selector: CSS選擇器
            wait_time: 等待頁面加載的時間（秒）
            
        Returns:
            Tuple: (原始內(nèi)容, 提取的內(nèi)容)
        """
        if not self.driver:
            self.setup_selenium_driver()
        
        try:
            self.driver.get(url)
            time.sleep(wait_time)  # 等待JavaScript執(zhí)行
            
            raw_content = self.driver.page_source
            
            # 提取特定部分
            extracted_content = None
            if css_selector:
                try:
                    elements = self.driver.find_elements(By.CSS_SELECTOR, css_selector)
                    extracted_content = '\n'.join([element.text for element in elements])
                except Exception as e:
                    self.logger.warning(f"提取元素失敗 {url}: {str(e)}")
            
            return raw_content, extracted_content
            
        except Exception as e:
            self.logger.error(f"Selenium獲取網(wǎng)頁失敗 {url}: {str(e)}")
            return None, None
    
    def take_screenshot(self, url: str, screenshot_path: str = None) -> Optional[str]:
        """
        截取網(wǎng)頁截圖
        
        Args:
            url: 網(wǎng)頁URL
            screenshot_path: 截圖保存路徑
            
        Returns:
            str: 截圖文件路徑
        """
        if not self.driver:
            self.setup_selenium_driver()
        
        try:
            self.driver.get(url)
            time.sleep(3)  # 等待頁面加載
            
            if screenshot_path is None:
                screenshot_dir = Path("screenshots")
                screenshot_dir.mkdir(exist_ok=True)
                filename = f"screenshot_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
                screenshot_path = screenshot_dir / filename
            
            self.driver.save_screenshot(str(screenshot_path))
            self.logger.info(f"截圖已保存: {screenshot_path}")
            return str(screenshot_path)
            
        except Exception as e:
            self.logger.error(f"截圖失敗 {url}: {str(e)}")
            return None
    
    def compare_screenshots(self, old_screenshot: str, new_screenshot: str) -> float:
        """
        比較兩張截圖的相似度
        
        Args:
            old_screenshot: 舊截圖路徑
            new_screenshot: 新截圖路徑
            
        Returns:
            float: 相似度（0-1）
        """
        try:
            from PIL import Image
            import imagehash
            
            # 計(jì)算圖片哈希
            old_hash = imagehash.average_hash(Image.open(old_screenshot))
            new_hash = imagehash.average_hash(Image.open(new_screenshot))
            
            # 計(jì)算相似度
            similarity = 1 - (old_hash - new_hash) / len(old_hash.hash) ** 2
            return similarity
            
        except Exception as e:
            self.logger.error(f"比較截圖失敗: {str(e)}")
            return 0.0
    
    def monitor_visual_changes(self, url: str, similarity_threshold: float = 0.95) -> bool:
        """
        監(jiān)控視覺變化
        
        Args:
            url: 網(wǎng)頁URL
            similarity_threshold: 相似度閾值
            
        Returns:
            bool: 是否檢測到顯著變化
        """
        # 獲取當(dāng)前截圖
        current_screenshot = self.take_screenshot(url)
        if not current_screenshot:
            return False
        
        # 獲取上一次的截圖
        previous_screenshot = self.get_previous_screenshot(url)
        
        if not previous_screenshot:
            # 第一次監(jiān)控，保存基準(zhǔn)截圖
            self.save_screenshot_reference(url, current_screenshot)
            return False
        
        # 比較截圖
        similarity = self.compare_screenshots(previous_screenshot, current_screenshot)
        
        if similarity < similarity_threshold:
            self.logger.info(f"檢測到視覺變化: {url}, 相似度: {similarity:.3f}")
            # 保存變化記錄
            self.record_visual_change(url, similarity, current_screenshot)
            return True
        
        return False
    
    def get_previous_screenshot(self, url: str) -> Optional[str]:
        """
        獲取上一次的截圖路徑
        
        Args:
            url: 網(wǎng)站URL
            
        Returns:
            str: 截圖路徑
        """
        # 這里需要實(shí)現(xiàn)從數(shù)據(jù)庫或文件系統(tǒng)獲取上一次的截圖
        # 簡化實(shí)現(xiàn)：在screenshots目錄中查找最新的截圖
        screenshot_dir = Path("screenshots")
        if not screenshot_dir.exists():
            return None
        
        # 查找該網(wǎng)站的最新截圖
        pattern = f"screenshot_*_{url.replace('://', '_').replace('/', '_')}.png"
        screenshots = list(screenshot_dir.glob(pattern))
        
        if screenshots:
            return str(max(screenshots, key=os.path.getctime))
        
        return None
    
    def save_screenshot_reference(self, url: str, screenshot_path: str):
        """
        保存截圖基準(zhǔn)
        
        Args:
            url: 網(wǎng)站URL
            screenshot_path: 截圖路徑
        """
        # 重命名文件以包含URL信息
        new_filename = f"reference_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{url.replace('://', '_').replace('/', '_')}.png"
        new_path = Path("screenshots") / new_filename
        
        try:
            os.rename(screenshot_path, new_path)
            self.logger.info(f"截圖基準(zhǔn)已保存: {new_path}")
        except Exception as e:
            self.logger.error(f"保存截圖基準(zhǔn)失敗: {str(e)}")
    
    def record_visual_change(self, url: str, similarity: float, screenshot_path: str):
        """
        記錄視覺變化
        
        Args:
            url: 網(wǎng)站URL
            similarity: 相似度
            screenshot_path: 截圖路徑
        """
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                INSERT INTO website_changes 
                (website_url, change_type, change_description, previous_content_hash, current_content_hash)
                VALUES (?, ?, ?, ?, ?)
            ''', (url, 'visual_change', f'視覺變化檢測，相似度: {similarity:.3f}', None, None))
            self.conn.commit()
            
            # 發(fā)送通知
            self.send_notification(url, f"檢測到視覺變化，相似度: {similarity:.3f}")
            
        except Exception as e:
            self.logger.error(f"記錄視覺變化失敗: {str(e)}")
    
    def cleanup(self):
        """清理資源"""
        if self.driver:
            self.driver.quit()
        super().cleanup()

# 使用示例
def demo_advanced_monitor():
    """演示高級監(jiān)控功能"""
    monitor = AdvancedWebsiteMonitor()
    
    # 添加需要JavaScript渲染的網(wǎng)站
    print("=== 添加JavaScript網(wǎng)站監(jiān)控 ===")
    monitor.add_website(
        "https://example.com",
        check_interval=600,
        css_selector=".dynamic-content"  # 監(jiān)控動態(tài)加載的內(nèi)容
    )
    
    # 測試視覺監(jiān)控
    print("\n=== 測試視覺監(jiān)控 ===")
    visual_change = monitor.monitor_visual_changes("https://example.com")
    print(f"視覺變化檢測: {'是' if visual_change else '否'}")
    
    # 使用Selenium獲取內(nèi)容
    print("\n=== 使用Selenium獲取內(nèi)容 ===")
    content, extracted = monitor.fetch_web_content_selenium(
        "https://example.com", 
        "h1"
    )
    
    if extracted:
        print(f"提取的內(nèi)容: {extracted[:100]}...")
    
    return monitor

if __name__ == "__main__":
    demo_advanced_monitor()

class SmartChangeDetector:
    """
    智能變化檢測器 - 使用多種策略提高檢測準(zhǔn)確性
    """
    
    def __init__(self):
        self.setup_logging()
    
    def setup_logging(self):
        """設(shè)置日志記錄"""
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s'
        )
        self.logger = logging.getLogger(__name__)
    
    def detect_significant_changes(self, old_content: str, new_content: str, 
                                 content_type: str = "general") -> Tuple[bool, str]:
        """
        檢測顯著性變化
        
        Args:
            old_content: 舊內(nèi)容
            new_content: 新內(nèi)容
            content_type: 內(nèi)容類型（"general", "price", "news"等）
            
        Returns:
            Tuple: (是否顯著變化, 變化描述)
        """
        if not old_content or not new_content:
            return False, "內(nèi)容為空，無法比較"
        
        # 基礎(chǔ)哈希比較
        old_hash = hashlib.md5(old_content.encode()).hexdigest()
        new_hash = hashlib.md5(new_content.encode()).hexdigest()
        
        if old_hash == new_hash:
            return False, "內(nèi)容完全一致"
        
        # 根據(jù)內(nèi)容類型使用不同的檢測策略
        if content_type == "price":
            return self.detect_price_changes(old_content, new_content)
        elif content_type == "news":
            return self.detect_news_changes(old_content, new_content)
        else:
            return self.detect_general_changes(old_content, new_content)
    
    def detect_price_changes(self, old_content: str, new_content: str) -> Tuple[bool, str]:
        """
        檢測價(jià)格變化
        
        Args:
            old_content: 舊內(nèi)容
            new_content: 新內(nèi)容
            
        Returns:
            Tuple: (是否價(jià)格變化, 變化描述)
        """
        # 提取價(jià)格信息（簡化實(shí)現(xiàn)）
        old_prices = self.extract_prices(old_content)
        new_prices = self.extract_prices(new_content)
        
        if not old_prices and not new_prices:
            return True, "檢測到變化但無法提取價(jià)格信息"
        
        changes = []
        
        # 比較價(jià)格
        for i, (old_price, new_price) in enumerate(zip(old_prices, new_prices)):
            if old_price != new_price:
                change_percent = ((new_price - old_price) / old_price) * 100
                changes.append(f"價(jià)格{i+1}: {old_price} → {new_price} ({change_percent:+.2f}%)")
        
        if changes:
            return True, "價(jià)格變化:\n" + "\n".join(changes)
        else:
            return True, "檢測到變化但不是價(jià)格變化"
    
    def extract_prices(self, content: str) -> List[float]:
        """
        從文本中提取價(jià)格
        
        Args:
            content: 文本內(nèi)容
            
        Returns:
            List: 價(jià)格列表
        """
        import re
        
        # 簡單的價(jià)格匹配模式
        price_pattern = r'[\$€￥￡]?(\d+[.,]\d{2})|\b(\d+)[.,](\d{2})\b'
        matches = re.findall(price_pattern, content)
        
        prices = []
        for match in matches:
            # 處理不同的價(jià)格格式
            price_str = ''.join(match).replace(',', '.')
            try:
                price = float(price_str)
                prices.append(price)
            except ValueError:
                continue
        
        return prices
    
    def detect_news_changes(self, old_content: str, new_content: str) -> Tuple[bool, str]:
        """
        檢測新聞內(nèi)容變化
        
        Args:
            old_content: 舊內(nèi)容
            new_content: 新內(nèi)容
            
        Returns:
            Tuple: (是否新聞變化, 變化描述)
        """
        # 分割為段落
        old_paragraphs = [p.strip() for p in old_content.split('\n') if p.strip()]
        new_paragraphs = [p.strip() for p in new_content.split('\n') if p.strip()]
        
        # 查找新增的段落
        new_paragraphs_set = set(new_paragraphs)
        old_paragraphs_set = set(old_paragraphs)
        
        added_paragraphs = new_paragraphs_set - old_paragraphs_set
        removed_paragraphs = old_paragraphs_set - new_paragraphs_set
        
        changes = []
        
        if added_paragraphs:
            changes.append(f"新增 {len(added_paragraphs)} 個段落")
            # 顯示前幾個新增段落
            for i, para in enumerate(list(added_paragraphs)[:3]):
                changes.append(f"  新增{i+1}: {para[:100]}...")
        
        if removed_paragraphs:
            changes.append(f"刪除 {len(removed_paragraphs)} 個段落")
        
        if changes:
            return True, "新聞內(nèi)容更新:\n" + "\n".join(changes)
        else:
            return True, "檢測到變化但無法識別具體更新"
    
    def detect_general_changes(self, old_content: str, new_content: str) -> Tuple[bool, str]:
        """
        檢測一般內(nèi)容變化
        
        Args:
            old_content: 舊內(nèi)容
            new_content: 新內(nèi)容
            
        Returns:
            Tuple: (是否顯著變化, 變化描述)
        """
        # 計(jì)算變化比例
        diff_ratio = self.calculate_change_ratio(old_content, new_content)
        
        if diff_ratio < 0.01:  # 1% 的變化閾值
            return False, f"微小變化 ({diff_ratio:.2%})"
        elif diff_ratio < 0.1:  # 10% 的變化閾值
            return True, f"中等變化 ({diff_ratio:.2%})"
        else:
            return True, f"重大變化 ({diff_ratio:.2%})"
    
    def calculate_change_ratio(self, old_content: str, new_content: str) -> float:
        """
        計(jì)算內(nèi)容變化比例
        
        Args:
            old_content: 舊內(nèi)容
            new_content: 新內(nèi)容
            
        Returns:
            float: 變化比例 (0-1)
        """
        if not old_content or not new_content:
            return 1.0  # 如果任一內(nèi)容為空，認(rèn)為完全變化
        
        # 使用difflib計(jì)算相似度
        matcher = difflib.SequenceMatcher(None, old_content, new_content)
        similarity = matcher.ratio()
        
        return 1 - similarity  # 返回變化比例
    
    def is_seasonal_content(self, content: str) -> bool:
        """
        判斷是否為季節(jié)性內(nèi)容（如廣告、橫幅等）
        
        Args:
            content: 內(nèi)容文本
            
        Returns:
            bool: 是否為季節(jié)性內(nèi)容
        """
        seasonal_keywords = [
            '促銷', '特價(jià)', '優(yōu)惠', '廣告', 'banner', 'promotion',
            'sale', 'discount', 'limited', 'limited time'
        ]
        
        content_lower = content.lower()
        for keyword in seasonal_keywords:
            if keyword in content_lower:
                return True
        
        return False

# 集成智能檢測到監(jiān)控器
class SmartWebsiteMonitor(AdvancedWebsiteMonitor):
    """
    智能網(wǎng)站監(jiān)控器 - 集成智能變化檢測
    """
    
    def __init__(self, config_file: str = "monitor_config.json"):
        super().__init__(config_file)
        self.change_detector = SmartChangeDetector()
    
    def detect_changes(self, url: str, current_content: str, 
                      previous_hash: str = None) -> Tuple[bool, Optional[str], Optional[str]]:
        """
        使用智能檢測策略
        
        Args:
            url: 網(wǎng)站URL
            current_content: 當(dāng)前內(nèi)容
            previous_hash: 上一次的內(nèi)容哈希值
            
        Returns:
            Tuple: (是否變化, 當(dāng)前哈希值, 變化描述)
        """
        if not current_content:
            return False, None, "無法獲取當(dāng)前內(nèi)容"
        
        current_hash = self.calculate_content_hash(current_content)
        
        # 如果沒有歷史記錄，保存初始快照
        if previous_hash is None:
            self.save_content_snapshot(url, current_hash, current_content)
            return False, current_hash, "首次監(jiān)控，建立基準(zhǔn)"
        
        # 檢查是否為季節(jié)性內(nèi)容
        if self.change_detector.is_seasonal_content(current_content):
            self.logger.info(f"檢測到季節(jié)性內(nèi)容，可能不需要報(bào)警: {url}")
            # 仍然保存快照但不觸發(fā)報(bào)警
            self.save_content_snapshot(url, current_hash, current_content)
            return False, current_hash, "季節(jié)性內(nèi)容變化"
        
        # 獲取上一次的內(nèi)容
        previous_content = self.get_previous_content_text(url)
        
        if not previous_content:
            self.save_content_snapshot(url, current_hash, current_content)
            return False, current_hash, "無法獲取歷史內(nèi)容"
        
        # 使用智能檢測
        content_type = self.determine_content_type(url)
        significant_change, change_description = self.change_detector.detect_significant_changes(
            previous_content, current_content, content_type
        )
        
        # 保存新快照
        self.save_content_snapshot(url, current_hash, current_content)
        
        return significant_change, current_hash, change_description
    
    def determine_content_type(self, url: str) -> str:
        """
        根據(jù)URL判斷內(nèi)容類型
        
        Args:
            url: 網(wǎng)站URL
            
        Returns:
            str: 內(nèi)容類型
        """
        # 簡單的URL模式匹配
        if any(keyword in url.lower() for keyword in ['amazon', 'taobao', 'jd', 'shop']):
            return "price"
        elif any(keyword in url.lower() for keyword in ['news', 'blog', 'article']):
            return "news"
        else:
            return "general"

# 使用示例
def demo_smart_monitor():
    """演示智能監(jiān)控功能"""
    monitor = SmartWebsiteMonitor()
    
    # 添加不同類型網(wǎng)站的監(jiān)控
    print("=== 添加智能監(jiān)控網(wǎng)站 ===")
    
    # 電商網(wǎng)站（價(jià)格監(jiān)控）
    monitor.add_website(
        "https://httpbin.org/json",  # 模擬電商網(wǎng)站
        check_interval=300,
        css_selector=None
    )
    
    # 新聞網(wǎng)站
    monitor.add_website(
        "https://example.com",  # 模擬新聞網(wǎng)站
        check_interval=600,
        css_selector="p"  # 監(jiān)控段落內(nèi)容
    )
    
    # 執(zhí)行檢查
    print("\n=== 執(zhí)行智能檢查 ===")
    results = monitor.check_all_websites()
    
    for result in results:
        status = "顯著變化" if result['changed'] else "無顯著變化"
        print(f"網(wǎng)站: {result['url']} - {status}")
        if result.get('change_description'):
            print(f"  分析: {result['change_description']}")
    
    return monitor

if __name__ == "__main__":
    demo_smart_monitor()

"""
網(wǎng)站變化監(jiān)控系統(tǒng) - 完整代碼實(shí)現(xiàn)
日期: 2024年
"""

import requests
from bs4 import BeautifulSoup
import hashlib
import time
import smtplib
import schedule
import json
import sqlite3
from datetime import datetime
import logging
from typing import Dict, List, Optional, Tuple
import difflib
import os
from pathlib import Path
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart

# Selenium相關(guān)導(dǎo)入（可選）
try:
    from selenium import webdriver
    from selenium.webdriver.chrome.options import Options
    from selenium.webdriver.chrome.service import Service
    from selenium.webdriver.common.by import By
    from webdriver_manager.chrome import ChromeDriverManager
    SELENIUM_AVAILABLE = True
except ImportError:
    SELENIUM_AVAILABLE = False

# 圖像處理相關(guān)導(dǎo)入（可選）
try:
    from PIL import Image
    import imagehash
    IMAGE_PROCESSING_AVAILABLE = True
except ImportError:
    IMAGE_PROCESSING_AVAILABLE = False

class WebsiteMonitor:
    """
    網(wǎng)站變化監(jiān)控器 - 完整實(shí)現(xiàn)
    """
    
    def __init__(self, config_file: str = "monitor_config.json"):
        self.config_file = Path(config_file)
        self.config = self.load_config()
        self.setup_logging()
        self.setup_database()
        self.driver = None
        
    def setup_logging(self):
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler('website_monitor.log', encoding='utf-8'),
                logging.StreamHandler()
            ]
        )
        self.logger = logging.getLogger(__name__)
    
    def load_config(self) -> Dict:
        default_config = {
            "monitoring_interval": 300,
            "notification_methods": {
                "email": {
                    "enabled": False,
                    "smtp_server": "smtp.gmail.com",
                    "smtp_port": 587,
                    "username": "your_email@gmail.com",
                    "password": "your_password",
                    "recipient": "recipient@example.com"
                },
                "webhook": {
                    "enabled": False,
                    "url": "https://your-webhook-url.com"
                },
                "telegram": {
                    "enabled": False,
                    "bot_token": "your_bot_token",
                    "chat_id": "your_chat_id"
                }
            },
            "websites": {},
            "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
            "use_selenium": False,
            "headless_browser": True
        }
        
        try:
            if self.config_file.exists():
                with open(self.config_file, 'r', encoding='utf-8') as f:
                    loaded_config = json.load(f)
                    return self.merge_configs(default_config, loaded_config)
            else:
                self.save_config(default_config)
                return default_config
        except Exception as e:
            self.logger.error(f"加載配置文件失敗: {str(e)}")
            return default_config
    
    def merge_configs(self, default: Dict, user: Dict) -> Dict:
        result = default.copy()
        
        def deep_merge(default_dict, user_dict):
            for key, value in user_dict.items():
                if key in default_dict and isinstance(default_dict[key], dict) and isinstance(value, dict):
                    deep_merge(default_dict[key], value)
                else:
                    default_dict[key] = value
        
        deep_merge(result, user)
        return result
    
    def save_config(self, config: Dict = None):
        if config is None:
            config = self.config
        
        try:
            with open(self.config_file, 'w', encoding='utf-8') as f:
                json.dump(config, f, indent=2, ensure_ascii=False)
            self.logger.info("配置已保存")
        except Exception as e:
            self.logger.error(f"保存配置失敗: {str(e)}")
    
    def setup_database(self):
        try:
            self.db_path = Path("website_monitor.db")
            self.conn = sqlite3.connect(self.db_path)
            self.create_tables()
            self.logger.info("數(shù)據(jù)庫初始化完成")
        except Exception as e:
            self.logger.error(f"數(shù)據(jù)庫初始化失敗: {str(e)}")
    
    def create_tables(self):
        cursor = self.conn.cursor()
        
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS website_changes (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                website_url TEXT NOT NULL,
                change_type TEXT NOT NULL,
                change_description TEXT,
                previous_content_hash TEXT,
                current_content_hash TEXT,
                change_timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
                notified BOOLEAN DEFAULT 0
            )
        ''')
        
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS content_snapshots (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                website_url TEXT NOT NULL,
                content_hash TEXT NOT NULL,
                content_text TEXT,
                snapshot_timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
            )
        ''')
        
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS monitor_configs (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                website_url TEXT UNIQUE NOT NULL,
                check_interval INTEGER DEFAULT 300,
                css_selector TEXT,
                enabled BOOLEAN DEFAULT 1,
                last_checked DATETIME
            )
        ''')
        
        self.conn.commit()
    
    def setup_selenium_driver(self):
        """設(shè)置Selenium WebDriver"""
        if not SELENIUM_AVAILABLE:
            self.logger.warning("Selenium不可用，無法初始化瀏覽器驅(qū)動")
            return
        
        try:
            chrome_options = Options()
            if self.config.get('headless_browser', True):
                chrome_options.add_argument("--headless")
            chrome_options.add_argument("--no-sandbox")
            chrome_options.add_argument("--disable-dev-shm-usage")
            chrome_options.add_argument("--disable-gpu")
            chrome_options.add_argument("--window-size=1920,1080")
            
            self.driver = webdriver.Chrome(
                service=Service(ChromeDriverManager().install()),
                options=chrome_options
            )
            self.logger.info("Selenium WebDriver初始化完成")
        except Exception as e:
            self.logger.error(f"Selenium WebDriver初始化失敗: {str(e)}")
    
    def fetch_web_content(self, url: str, css_selector: str = None) -> Tuple[Optional[str], Optional[str]]:
        """獲取網(wǎng)頁內(nèi)容"""
        use_selenium = self.config.get('use_selenium', False)
        
        if use_selenium and SELENIUM_AVAILABLE:
            return self.fetch_web_content_selenium(url, css_selector)
        else:
            return self.fetch_web_content_requests(url, css_selector)
    
    def fetch_web_content_requests(self, url: str, css_selector: str = None) -> Tuple[Optional[str], Optional[str]]:
        """使用requests獲取網(wǎng)頁內(nèi)容"""
        try:
            headers = {
                'User-Agent': self.config.get('user_agent', 
                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
            }
            
            response = requests.get(url, headers=headers, timeout=30)
            response.raise_for_status()
            
            raw_content = response.text
            extracted_content = None
            
            if css_selector:
                soup = BeautifulSoup(raw_content, 'html.parser')
                selected_elements = soup.select(css_selector)
                if selected_elements:
                    extracted_content = '\n'.join([elem.get_text(strip=True) for elem in selected_elements])
            
            self.logger.debug(f"成功獲取網(wǎng)頁內(nèi)容: {url}")
            return raw_content, extracted_content
            
        except Exception as e:
            self.logger.error(f"獲取網(wǎng)頁內(nèi)容失敗 {url}: {str(e)}")
            return None, None
    
    def fetch_web_content_selenium(self, url: str, css_selector: str = None, 
                                 wait_time: int = 5) -> Tuple[Optional[str], Optional[str]]:
        """使用Selenium獲取網(wǎng)頁內(nèi)容"""
        if not self.driver:
            self.setup_selenium_driver()
        
        if not self.driver:
            self.logger.error("Selenium驅(qū)動未初始化")
            return None, None
        
        try:
            self.driver.get(url)
            time.sleep(wait_time)
            
            raw_content = self.driver.page_source
            extracted_content = None
            
            if css_selector:
                try:
                    elements = self.driver.find_elements(By.CSS_SELECTOR, css_selector)
                    extracted_content = '\n'.join([element.text for element in elements])
                except Exception as e:
                    self.logger.warning(f"提取元素失敗 {url}: {str(e)}")
            
            return raw_content, extracted_content
            
        except Exception as e:
            self.logger.error(f"Selenium獲取網(wǎng)頁失敗 {url}: {str(e)}")
            return None, None
    
    def calculate_content_hash(self, content: str) -> str:
        return hashlib.md5(content.encode('utf-8')).hexdigest()
    
    def save_content_snapshot(self, url: str, content_hash: str, content_text: str = None):
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                INSERT INTO content_snapshots (website_url, content_hash, content_text)
                VALUES (?, ?, ?)
            ''', (url, content_hash, content_text))
            self.conn.commit()
        except Exception as e:
            self.logger.error(f"保存內(nèi)容快照失敗: {str(e)}")
    
    def get_previous_content_hash(self, url: str) -> Optional[str]:
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                SELECT content_hash FROM content_snapshots 
                WHERE website_url = ? 
                ORDER BY snapshot_timestamp DESC 
                LIMIT 1
            ''', (url,))
            result = cursor.fetchone()
            return result[0] if result else None
        except Exception as e:
            self.logger.error(f"獲取歷史哈希值失敗 {url}: {str(e)}")
            return None
    
    def detect_changes(self, url: str, current_content: str, 
                      previous_hash: str = None) -> Tuple[bool, Optional[str], Optional[str]]:
        if not current_content:
            return False, None, "無法獲取當(dāng)前內(nèi)容"
        
        current_hash = self.calculate_content_hash(current_content)
        
        if previous_hash is None:
            self.save_content_snapshot(url, current_hash, current_content)
            return False, current_hash, "首次監(jiān)控，建立基準(zhǔn)"
        
        if current_hash == previous_hash:
            return False, current_hash, "內(nèi)容未變化"
        
        previous_content = self.get_previous_content_text(url)
        change_description = self.analyze_content_changes(previous_content, current_content)
        
        self.save_content_snapshot(url, current_hash, current_content)
        return True, current_hash, change_description
    
    def get_previous_content_text(self, url: str) -> Optional[str]:
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                SELECT content_text FROM content_snapshots 
                WHERE website_url = ? 
                ORDER BY snapshot_timestamp DESC 
                LIMIT 1
            ''', (url,))
            result = cursor.fetchone()
            return result[0] if result else None
        except Exception as e:
            self.logger.error(f"獲取歷史內(nèi)容失敗 {url}: {str(e)}")
            return None
    
    def analyze_content_changes(self, old_content: str, new_content: str) -> str:
        if not old_content or not new_content:
            return "無法比較內(nèi)容變化"
        
        diff = difflib.unified_diff(
            old_content.splitlines(keepends=True),
            new_content.splitlines(keepends=True),
            fromfile='舊內(nèi)容',
            tofile='新內(nèi)容',
            n=3
        )
        
        diff_text = ''.join(diff)
        
        if diff_text:
            lines = diff_text.split('\n')[:10]
            return "檢測到內(nèi)容變化:\n" + '\n'.join(lines)
        else:
            return "內(nèi)容發(fā)生變化但無法生成差異報(bào)告"
    
    def add_website(self, url: str, check_interval: int = 300, 
                   css_selector: str = None, enabled: bool = True) -> bool:
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                INSERT OR REPLACE INTO monitor_configs 
                (website_url, check_interval, css_selector, enabled, last_checked)
                VALUES (?, ?, ?, ?, ?)
            ''', (url, check_interval, css_selector, enabled, None))
            
            self.conn.commit()
            
            if 'websites' not in self.config:
                self.config['websites'] = {}
            
            self.config['websites'][url] = {
                'check_interval': check_interval,
                'css_selector': css_selector,
                'enabled': enabled
            }
            
            self.save_config()
            self.logger.info(f"已添加監(jiān)控網(wǎng)站: {url}")
            return True
            
        except Exception as e:
            self.logger.error(f"添加網(wǎng)站失敗 {url}: {str(e)}")
            return False
    
    def remove_website(self, url: str) -> bool:
        try:
            cursor = self.conn.cursor()
            cursor.execute('DELETE FROM monitor_configs WHERE website_url = ?', (url,))
            self.conn.commit()
            
            if url in self.config.get('websites', {}):
                del self.config['websites'][url]
                self.save_config()
            
            self.logger.info(f"已移除監(jiān)控網(wǎng)站: {url}")
            return True
            
        except Exception as e:
            self.logger.error(f"移除網(wǎng)站失敗 {url}: {str(e)}")
            return False
    
    def check_website(self, url: str) -> Dict:
        self.logger.info(f"檢查網(wǎng)站: {url}")
        
        website_config = self.config['websites'].get(url, {})
        css_selector = website_config.get('css_selector')
        
        raw_content, extracted_content = self.fetch_web_content(url, css_selector)
        content_to_check = extracted_content if extracted_content else raw_content
        
        if not content_to_check:
            return {
                'url': url,
                'changed': False,
                'error': '無法獲取網(wǎng)頁內(nèi)容',
                'timestamp': datetime.now()
            }
        
        previous_hash = self.get_previous_content_hash(url)
        changed, current_hash, change_description = self.detect_changes(
            url, content_to_check, previous_hash
        )
        
        result = {
            'url': url,
            'changed': changed,
            'current_hash': current_hash,
            'previous_hash': previous_hash,
            'change_description': change_description,
            'timestamp': datetime.now()
        }
        
        if changed:
            self.logger.info(f"檢測到變化: {url}")
            self.record_change(url, change_description, previous_hash, current_hash)
            self.send_notification(url, change_description)
        
        return result
    
    def record_change(self, url: str, change_description: str, 
                     previous_hash: str, current_hash: str):
        try:
            cursor = self.conn.cursor()
            cursor.execute('''
                INSERT INTO website_changes 
                (website_url, change_type, change_description, previous_content_hash, current_content_hash)
                VALUES (?, ?, ?, ?, ?)
            ''', (url, 'content_change', change_description, previous_hash, current_hash))
            self.conn.commit()
        except Exception as e:
            self.logger.error(f"記錄變化失敗: {str(e)}")
    
    def send_notification(self, url: str, change_description: str):
        notification_methods = self.config.get('notification_methods', {})
        
        if notification_methods.get('email', {}).get('enabled', False):
            self.send_email_notification(url, change_description)
        
        if notification_methods.get('webhook', {}).get('enabled', False):
            self.send_webhook_notification(url, change_description)
        
        if notification_methods.get('telegram', {}).get('enabled', False):
            self.send_telegram_notification(url, change_description)
    
    def send_email_notification(self, url: str, change_description: str):
        try:
            email_config = self.config['notification_methods']['email']
            
            subject = f"網(wǎng)站變化通知: {url}"
            body = f"""
檢測到網(wǎng)站內(nèi)容發(fā)生變化:

網(wǎng)站: {url}
時間: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

變化詳情:
{change_description}

---
此通知由網(wǎng)站監(jiān)控系統(tǒng)自動發(fā)送
            """
            
            self._send_email(
                email_config['smtp_server'],
                email_config['smtp_port'],
                email_config['username'],
                email_config['password'],
                email_config['recipient'],
                subject,
                body
            )
            
            self.logger.info(f"已發(fā)送郵件通知: {url}")
            
        except Exception as e:
            self.logger.error(f"發(fā)送郵件通知失敗: {str(e)}")
    
    def _send_email(self, smtp_server: str, port: int, username: str, 
                   password: str, recipient: str, subject: str, body: str):
        """發(fā)送郵件具體實(shí)現(xiàn)"""
        try:
            msg = MIMEMultipart()
            msg['From'] = username
            msg['To'] = recipient
            msg['Subject'] = subject
            
            msg.attach(MIMEText(body, 'plain', 'utf-8'))
            
            server = smtplib.SMTP(smtp_server, port)
            server.starttls()
            server.login(username, password)
            server.send_message(msg)
            server.quit()
            
        except Exception as e:
            self.logger.error(f"郵件發(fā)送失敗: {str(e)}")
            raise
    
    def send_webhook_notification(self, url: str, change_description: str):
        try:
            webhook_config = self.config['notification_methods']['webhook']
            webhook_url = webhook_config['url']
            
            payload = {
                'url': url,
                'change_description': change_description,
                'timestamp': datetime.now().isoformat(),
                'type': 'website_change'
            }
            
            response = requests.post(webhook_url, json=payload, timeout=10)
            response.raise_for_status()
            
            self.logger.info(f"已發(fā)送Webhook通知: {url}")
            
        except Exception as e:
            self.logger.error(f"發(fā)送Webhook通知失敗: {str(e)}")
    
    def send_telegram_notification(self, url: str, change_description: str):
        try:
            telegram_config = self.config['notification_methods']['telegram']
            bot_token = telegram_config['bot_token']
            chat_id = telegram_config['chat_id']
            
            message = f"""
?? 網(wǎng)站變化通知

?? 網(wǎng)站: {url}
? 時間: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

?? 變化詳情:
{change_description}

#網(wǎng)站監(jiān)控
            """
            
            api_url = f"https://api.telegram.org/bot{bot_token}/sendMessage"
            payload = {
                'chat_id': chat_id,
                'text': message,
                'parse_mode': 'HTML'
            }
            
            response = requests.post(api_url, json=payload, timeout=10)
            response.raise_for_status()
            
            self.logger.info(f"已發(fā)送Telegram通知: {url}")
            
        except Exception as e:
            self.logger.error(f"發(fā)送Telegram通知失敗: {str(e)}")
    
    def check_all_websites(self) -> List[Dict]:
        results = []
        websites = self.config.get('websites', {})
        
        for url, config in websites.items():
            if config.get('enabled', True):
                try:
                    result = self.check_website(url)
                    results.append(result)
                    time.sleep(1)  # 避免請求過于頻繁
                except Exception as e:
                    self.logger.error(f"檢查網(wǎng)站失敗 {url}: {str(e)}")
                    results.append({
                        'url': url,
                        'changed': False,
                        'error': str(e),
                        'timestamp': datetime.now()
                    })
        
        return results
    
    def start_monitoring(self):
        self.logger.info("啟動網(wǎng)站監(jiān)控服務(wù)")
        
        interval = self.config.get('monitoring_interval', 300)
        schedule.every(interval).seconds.do(self.check_all_websites)
        
        self.check_all_websites()
        
        try:
            while True:
                schedule.run_pending()
                time.sleep(1)
        except KeyboardInterrupt:
            self.logger.info("監(jiān)控服務(wù)已停止")
        finally:
            self.cleanup()
    
    def cleanup(self):
        if self.driver:
            self.driver.quit()
        if hasattr(self, 'conn'):
            self.conn.close()
        self.logger.info("資源清理完成")

def main():
    """主函數(shù) - 演示完整功能"""
    monitor = WebsiteMonitor()
    
    print("=== 網(wǎng)站變化監(jiān)控系統(tǒng) ===")
    print("請選擇操作:")
    print("1. 添加監(jiān)控網(wǎng)站")
    print("2. 移除監(jiān)控網(wǎng)站")
    print("3. 立即檢查所有網(wǎng)站")
    print("4. 開始自動監(jiān)控")
    print("5. 查看監(jiān)控統(tǒng)計(jì)")
    print("6. 退出")
    
    while True:
        choice = input("\n請輸入選擇 (1-6): ").strip()
        
        if choice == '1':
            url = input("請輸入網(wǎng)站URL: ").strip()
            interval = input("檢查間隔(秒，默認(rèn)300): ").strip()
            interval = int(interval) if interval else 300
            selector = input("CSS選擇器(可選): ").strip()
            selector = selector if selector else None
            
            success = monitor.add_website(url, interval, selector)
            if success:
                print(f"成功添加網(wǎng)站: {url}")
            else:
                print("添加網(wǎng)站失敗")
        
        elif choice == '2':
            url = input("請輸入要移除的網(wǎng)站URL: ").strip()
            success = monitor.remove_website(url)
            if success:
                print(f"成功移除網(wǎng)站: {url}")
            else:
                print("移除網(wǎng)站失敗")
        
        elif choice == '3':
            print("開始檢查所有網(wǎng)站...")
            results = monitor.check_all_websites()
            print(f"檢查完成，共檢查 {len(results)} 個網(wǎng)站")
            
            changes = [r for r in results if r['changed']]
            if changes:
                print(f"發(fā)現(xiàn) {len(changes)} 個網(wǎng)站有變化:")
                for result in changes:
                    print(f"  - {result['url']}")
                    print(f"    變化: {result['change_description'][:100]}...")
            else:
                print("所有網(wǎng)站均無變化")
        
        elif choice == '4':
            print("啟動自動監(jiān)控服務(wù)...")
            print("按 Ctrl+C 停止監(jiān)控")
            monitor.start_monitoring()
        
        elif choice == '5':
            websites = monitor.config.get('websites', {})
            print(f"監(jiān)控網(wǎng)站數(shù)量: {len(websites)}")
            print("當(dāng)前監(jiān)控的網(wǎng)站:")
            for url, config in websites.items():
                status = "啟用" if config.get('enabled', True) else "禁用"
                print(f"  - {url} ({status})")
        
        elif choice == '6':
            print("謝謝使用！")
            break
        
        else:
            print("無效選擇，請重新輸入")

if __name__ == "__main__":
    main()