52 lines
1.4 KiB
Python
52 lines
1.4 KiB
Python
"""
|
|
Logging utility for web scraping operations.
|
|
"""
|
|
import logging
|
|
import sys
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
from config import LOGS_DIR
|
|
|
|
|
|
def setup_logger(name: str, level: int = logging.INFO) -> logging.Logger:
|
|
"""
|
|
Set up a logger with both file and console handlers.
|
|
|
|
Args:
|
|
name: Logger name (typically __name__ of the calling module)
|
|
level: Logging level (default: INFO)
|
|
|
|
Returns:
|
|
Configured logger instance
|
|
"""
|
|
logger = logging.getLogger(name)
|
|
logger.setLevel(level)
|
|
|
|
# Avoid duplicate handlers
|
|
if logger.handlers:
|
|
return logger
|
|
|
|
# Create formatters
|
|
detailed_formatter = logging.Formatter(
|
|
'%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
datefmt='%Y-%m-%d %H:%M:%S'
|
|
)
|
|
simple_formatter = logging.Formatter('%(levelname)s - %(message)s')
|
|
|
|
# File handler - detailed logs
|
|
log_file = LOGS_DIR / f"{datetime.now().strftime('%Y%m%d')}_scraping.log"
|
|
file_handler = logging.FileHandler(log_file, encoding='utf-8')
|
|
file_handler.setLevel(logging.DEBUG)
|
|
file_handler.setFormatter(detailed_formatter)
|
|
|
|
# Console handler - simplified logs
|
|
console_handler = logging.StreamHandler(sys.stdout)
|
|
console_handler.setLevel(level)
|
|
console_handler.setFormatter(simple_formatter)
|
|
|
|
# Add handlers
|
|
logger.addHandler(file_handler)
|
|
logger.addHandler(console_handler)
|
|
|
|
return logger
|
|
|