madomeda/core/processor.py

"""
Main frontmatter processor
"""
import sys
from pathlib import Path
from datetime import datetime
from core.database import Database
from core.repository import Repository
from core.frontmatter import FrontmatterParser
from core.rules_processor import RulesProcessor
from core.template import TemplateManager
from core.llm import LLMClient
from core.changelog import ChangeLog
from core.config import ConfigManager


class FrontmatterProcessor:
    def __init__(self, repo: Repository, template_name: str, 
                 whatif: bool = False, no_confirm: bool = False, force: bool = False, add_only: bool = False):
        self.repo = repo
        self.template_name = template_name
        self.whatif = whatif
        self.no_confirm = no_confirm
        self.force = force
        self.add_only = add_only
        
        # Initialize configuration
        self.config = ConfigManager()
        self.config.ensure_config_exists()
        
        # Get API configuration
        api_config = self.config.get_api_config()
        
        self.db = Database(repo.path / 'madomeda.db')
        self.rules = RulesProcessor(self.config.rules_dir)
        self.templates = TemplateManager(self.config.templates_dir)
        self.llm = LLMClient(self.config.prompts_dir, api_config)
        self.changelog = ChangeLog(repo.path)
        self.parser = FrontmatterParser()
    
    def process(self):
        """Main processing loop"""
        # Print configuration info if running in interactive shell
        if sys.stdin.isatty():
            self.config.print_config_info()
        
        print(f"Processing repository: {self.repo.path}")
        print(f"Template: {self.template_name}")
        print(f"Mode: {'DRY RUN' if self.whatif else 'LIVE'}")
        if self.add_only:
            print(f"Add-only mode: Preserving existing keys")
        print()
        
        # Get template
        template = self.templates.load_template(self.template_name)
        schema = self.templates.get_structured_output_schema(template)
        
        # Get all markdown files
        files = self.repo.get_tracked_files()
        print(f"Found {len(files)} markdown files")
        print()
        
        # Start changelog session
        self.changelog.start_session()
        
        # Process each file
        for file_path in files:
            self._process_file(file_path, template, schema)
        
        # Collect all tags and store in database
        self._collect_all_tags()
        
        # Write changelog
        if not self.whatif:
            self.changelog.write()
        
        self.db.close()
    
    def _process_file(self, file_path: Path, template: dict, schema: dict):
        """Process a single markdown file"""
        relative_path = file_path.relative_to(self.repo.path)
        print(f"Processing: {relative_path}")
        
        # Add file to database
        file_id = self.db.add_file(str(relative_path))
        
        # Read file content
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()
        
        # Parse frontmatter
        original_fm, body = self.parser.parse(content)
        
        if original_fm:
            print(f"  Found existing frontmatter")
        else:
            print(f"  No frontmatter found")
            original_fm = {}
        
        # Store original frontmatter in database
        if original_fm:
            self.db.add_frontmatter(file_id, original_fm, conformant=False)
        
        # Get git history
        git_info = self.repo.get_file_history(file_path)
        self.db.add_commit_info(
            file_id,
            git_info['author_name'],
            git_info['author_email'],
            git_info['commit_hash'],
            git_info['commit_tag'],
            git_info['latest_hash']
        )
        
        # Apply rules
        normalized_fm, is_conformant, violations = self.rules.apply_rules(dict(original_fm))
        
        # Check if we need to update
        needs_update = self.force or not is_conformant or not self._matches_template(normalized_fm, template)
        
        if not needs_update:
            print(f"  OK Already conformant")
            if original_fm:
                # Mark as conformant in database
                fm_id = self.db.add_frontmatter(file_id, original_fm, conformant=True)
            print()
            return
        
        # Build new frontmatter from template
        new_fm = self._build_frontmatter(template, schema, original_fm, normalized_fm, body, file_path, git_info)
        
        # In add-only mode, merge with normalized original
        if self.add_only:
            # Start with normalized original
            merged_fm = dict(normalized_fm)
            # Add missing template fields
            for key, value in new_fm.items():
                if key not in merged_fm:
                    merged_fm[key] = value
            new_fm = merged_fm
        
        # Store tags in database
        if 'tags' in new_fm and isinstance(new_fm['tags'], list):
            for tag in new_fm['tags']:
                self.db.add_tag(tag)
        
        # Check for metadata loss
        discarded = self._check_discarded_metadata(original_fm, new_fm)
        if discarded and not self.add_only:  # Only warn in non-add-only mode
            print(f"  WARNING Metadata will be discarded: {', '.join(discarded)}")
            
            if not self.no_confirm and not self.whatif:
                response = input("  Continue? (y/n): ")
                if response.lower() != 'y':
                    print(f"  Skipped")
                    print()
                    return
        
        # Show what would change
        if self.whatif:
            print(f"  Would update frontmatter:")
            print(f"    Changes: {self._describe_changes(original_fm, new_fm)}")
        else:
            # Write new frontmatter
            new_content = self.parser.serialize(new_fm, body)
            with open(file_path, 'w', encoding='utf-8') as f:
                f.write(new_content)
            
            # Update database
            self.db.add_frontmatter(file_id, new_fm, conformant=True)
            
            # Add to changelog
            self.changelog.add_entry(str(relative_path))
            
            print(f"  OK Updated frontmatter")
        
        print()
    
    def _matches_template(self, frontmatter: dict, template: dict) -> bool:
        """Check if frontmatter has all required template fields"""
        for field in template.keys():
            if field not in frontmatter:
                return False
            # Allow empty lists/strings as valid values
        return True
    
    def _build_frontmatter(self, template: dict, schema: dict, original_fm: dict, 
                          normalized_fm: dict, body: str, file_path: Path, git_info: dict) -> dict:
        """Build new frontmatter from template"""
        new_fm = {}
        all_tags = self.db.get_all_tags()
        
        for field, field_value in template.items():
            # Parse field to get strategy and type
            strategy_str, field_type = self.templates.parse_field(str(field_value))
            options = strategy_str.split('|')
            value = None
            
            for option in options:
                option = option.strip()
                
                if option.startswith('lit:'):
                    # Literal/static value
                    value = option[4:].strip()
                    # Convert based on type
                    value = self._convert_value_type(value, field_type)
                elif option == 'heur':
                    # Use heuristics
                    value = self._get_heuristic_value(field, normalized_fm, body, file_path, git_info)
                elif option == 'orig':
                    # Use original value
                    value = normalized_fm.get(field)
                elif option.startswith('ai'):
                    # Use LLM
                    parts = option.split(None, 1)
                    prompt_name = parts[1] if len(parts) > 1 else 'default'
                    value = self.llm.infer_metadata(field, prompt_name, original_fm, new_fm, body, all_tags, schema)
                
                # Accept value if not None (empty lists/strings are valid)
                if value is not None:
                    break
            
            if value is not None:
                new_fm[field] = value
        
        return new_fm
    
    def _convert_value_type(self, value: str, field_type: str):
        """Convert literal value to appropriate type"""
        if field_type == 'checkbox':
            return value.lower() in ('true', '1', 'yes', 'on')
        elif field_type == 'number':
            try:
                if '.' in value:
                    return float(value)
                return int(value)
            except ValueError:
                return value
        elif field_type == 'list':
            # Parse as JSON array or return empty list
            if value == '[]':
                return []
            try:
                import json
                return json.loads(value)
            except:
                return [value]
        else:
            # text, date, datetime - keep as string
            return value
    
    def _get_heuristic_value(self, field: str, normalized_fm: dict, body: str, 
                            file_path: Path, git_info: dict):
        """Get field value using heuristics"""
        if field == 'title':
            # Try to extract from document
            title = self.parser.extract_title(body)
            if not title:
                # Use filename without extension
                title = file_path.stem
            return title
        
        elif field == 'created':
            return git_info.get('created')
        
        elif field == 'changed':
            return git_info.get('changed')
        
        elif field == 'authors':
            return git_info.get('authors', [])
        
        elif field == 'version':
            # Prefer latest tag, fallback to short hash
            latest_tag = git_info.get('latest_tag', '')
            if latest_tag:
                return latest_tag
            latest_hash = git_info.get('latest_hash', '')
            return latest_hash[:7] if latest_hash else ''
        
        elif field == 'tags':
            tags = normalized_fm.get('tags', [])
            # Return None if empty so AI inference is tried
            return tags if tags else None
        
        return None
    
    def _check_discarded_metadata(self, original: dict, new: dict) -> list:
        """Check for metadata that will be discarded"""
        discarded = []
        for key in original.keys():
            if key not in new:
                discarded.append(key)
        return discarded
    
    def _describe_changes(self, original: dict, new: dict) -> str:
        """Describe changes between frontmatter versions"""
        changes = []
        
        # New fields
        for key in new.keys():
            if key not in original:
                changes.append(f"+{key}")
        
        # Modified fields
        for key in new.keys():
            if key in original and original[key] != new[key]:
                changes.append(f"~{key}")
        
        # Removed fields
        for key in original.keys():
            if key not in new:
                changes.append(f"-{key}")
        
        return ', '.join(changes) if changes else 'none'
    
    def _collect_all_tags(self):
        """Collect all unique tags from processed files and store in database"""
        # Tags are collected during file processing
        all_tags = self.db.get_all_tags()
        print(f"Total unique tags in repository: {len(all_tags)}")