#!/usr/bin/env python3 """Quick test script for SS Crawler components.""" import sys from pathlib import Path def test_imports(): """Test if all required modules can be imported.""" print("๐Ÿ” Testing imports...") try: import scrapy print(f"โœ… Scrapy {scrapy.__version__}") except ImportError as e: print(f"โŒ Scrapy not found: {e}") print(" Install: pip install scrapy") return False try: import click print("โœ… Click") except ImportError as e: print(f"โŒ Click not found: {e}") print(" Install: pip install click") return False try: import pydantic print(f"โœ… Pydantic {pydantic.__version__}") except ImportError as e: print(f"โŒ Pydantic not found: {e}") print(" Install: pip install pydantic") return False try: import pandas print(f"โœ… Pandas {pandas.__version__}") except ImportError as e: print(f"โŒ Pandas not found: {e}") print(" Install: pip install pandas") return False try: import yaml print("โœ… PyYAML") except ImportError as e: print(f"โŒ PyYAML not found: {e}") print(" Install: pip install pyyaml") return False return True def test_project_structure(): """Test if project structure is correct.""" print("\n๐Ÿ” Testing project structure...") required_files = [ 'scrapy_project/__init__.py', 'scrapy_project/items.py', 'scrapy_project/spiders/__init__.py', 'scrapy_project/spiders/generic_spider.py', 'scrapy_project/pipelines.py', 'scrapy_project/middlewares.py', 'scrapy_project/settings.py', 'scrapy_project/utils/detector.py', 'main.py', 'scrapy.cfg', 'requirements.txt' ] all_ok = True for file_path in required_files: path = Path(file_path) if path.exists(): print(f"โœ… {file_path}") else: print(f"โŒ {file_path} - NOT FOUND") all_ok = False return all_ok def test_detector(): """Test website detector.""" print("\n๐Ÿ” Testing website detector...") try: from scrapy_project.utils.detector import WebsiteDetector detector = WebsiteDetector() # Test detection test_urls = [ 'https://shopee.vn', 'https://tiki.vn', 'https://example.com' ] for url in test_urls: config = detector.detect(url) if config: print(f"โœ… {url} -> {config.name}") else: print(f"โš ๏ธ {url} -> Not detected") return True except Exception as e: print(f"โŒ Detector test failed: {e}") import traceback traceback.print_exc() return False def test_items(): """Test items and models.""" print("\n๐Ÿ” Testing items and models...") try: from scrapy_project.items import ProductItem, Product # Test ProductItem item = ProductItem() item['name'] = 'Test Product' item['price'] = 100000 item['currency'] = 'VND' print("โœ… ProductItem created") # Test Product model product = Product.from_scrapy_item(item) print(f"โœ… Product model created: {product.name}") # Test export methods wc_row = product.to_woocommerce_csv_row() print(f"โœ… WooCommerce export: {wc_row['Name']}") shopify_row = product.to_shopify_csv_row() print(f"โœ… Shopify export: {shopify_row['Title']}") return True except Exception as e: print(f"โŒ Items test failed: {e}") import traceback traceback.print_exc() return False def main(): """Run all tests.""" print("=" * 50) print("๐Ÿงช SS Crawler - Quick Test") print("=" * 50) results = [] # Test imports results.append(("Imports", test_imports())) # Test structure results.append(("Project Structure", test_project_structure())) # Test detector (even without scrapy installed) results.append(("Website Detector", test_detector())) # Test items (even without scrapy installed) results.append(("Items & Models", test_items())) # Summary print("\n" + "=" * 50) print("๐Ÿ“Š Test Summary") print("=" * 50) passed = sum(1 for _, result in results if result) total = len(results) for name, result in results: status = "โœ… PASS" if result else "โŒ FAIL" print(f"{status} - {name}") print(f"\nTotal: {passed}/{total} tests passed") if passed == total: print("\n๐ŸŽ‰ All tests passed! Ready to use.") return 0 else: print("\nโš ๏ธ Some tests failed. Please install missing dependencies.") print("\nInstall dependencies:") print(" pip install -r requirements.txt") return 1 if __name__ == '__main__': sys.exit(main())