From eea7ef06483189150c465a6eedf4c648719e6af4 Mon Sep 17 00:00:00 2001 From: Test User Date: Thu, 23 Oct 2025 20:14:21 +0200 Subject: [PATCH] improvements added to rules engine --- ADD_ONLY_EXAMPLES.md | 8 +- CONFIGURATION.md | 11 + GETTING_STARTED.md | 24 +- IMPLEMENTATION.md | 8 +- LITERAL_VALUES.md | 342 ++++++++++++++++++++++++ PROJECT_SUMMARY.md | 8 +- README.md | 30 ++- RULES_GUIDE.md | 562 ++++++++++++++++++++++++++++++++++++++++ STRUCTURE.md | 8 +- TEMPLATE_TYPES.md | 470 +++++++++++++++++++++++++++++++++ USAGE.md | 8 +- VERSION_FIELD.md | 8 +- advanced.md | 8 +- ai_test.md | 8 +- comparison.md | 8 +- core/config.py | 111 ++++++-- core/processor.py | 43 ++- core/rules_processor.py | 255 ++++++++++++++++++ core/template.py | 65 ++++- guide.md | 8 +- madomeda_changelog.txt | 42 +++ rules_test.md | 15 +- sample.md | 8 +- test.md | 8 +- type_test.md | 8 + 25 files changed, 1967 insertions(+), 107 deletions(-) create mode 100644 LITERAL_VALUES.md create mode 100644 RULES_GUIDE.md create mode 100644 TEMPLATE_TYPES.md create mode 100644 core/rules_processor.py diff --git a/ADD_ONLY_EXAMPLES.md b/ADD_ONLY_EXAMPLES.md index db77331..88d0487 100644 --- a/ADD_ONLY_EXAMPLES.md +++ b/ADD_ONLY_EXAMPLES.md @@ -1,11 +1,11 @@ --- title: Add-Only Mode Examples created: '2025-10-23T18:33:48+02:00' -changed: '2025-10-23T18:33:48+02:00' -authors: -- Test User -version: 8c4f612 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Add-Only Mode Examples diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 610c47a..d747219 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -1,3 +1,14 @@ +--- +title: Configuration Management +created: '2025-10-23T19:29:26+02:00' +status: draft +priority: 5 +published: false +tags: +- aiml +- python +reviewers: [] +--- # Configuration Management ## Configuration Directory diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md index 3ae3d5d..5100dc7 100644 --- a/GETTING_STARTED.md +++ b/GETTING_STARTED.md @@ -1,11 +1,11 @@ --- title: Getting Started with Madomeda created: '2025-10-23T18:33:48+02:00' -changed: '2025-10-23T18:33:48+02:00' -authors: -- Test User -version: 8c4f612 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Getting Started with Madomeda @@ -143,16 +143,23 @@ python madomeda.py --template minimal ### Add Custom Rule -1. Create `rules/my_rule.json`: +See [RULES_GUIDE.md](RULES_GUIDE.md) for comprehensive rule writing documentation. + +1. Create `~/.config/madomeda/rules/60_my_rule.json`: ```json { "name": "my_custom_rule", "description": "Description of what this rule does", - "type": "custom" + "field": "my_field", + "priority": 60, + "action": "normalize_value", + "pattern": "[^a-z]", + "replacement": "", + "llm_prompt": "Instructions for LLM" } ``` -2. Implement logic in `core/rules.py` (requires code modification) +2. Rules are automatically loaded - no code changes needed! ### Customize LLM Prompt @@ -214,6 +221,9 @@ python madomeda.py --template minimal 1. Check documentation: - `README.md` - Overview - `USAGE.md` - Detailed examples + - `RULES_GUIDE.md` - Writing custom rules + - `LITERAL_VALUES.md` - Template literal values + - `CONFIGURATION.md` - Configuration options - `STRUCTURE.md` - Architecture - `IMPLEMENTATION.md` - Technical details diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 9a4016f..a3348ee 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -1,11 +1,11 @@ --- title: Madomeda - Implementation Summary created: '2025-10-23T18:33:48+02:00' -changed: '2025-10-23T18:33:48+02:00' -authors: -- Test User -version: 8c4f612 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Madomeda - Implementation Summary diff --git a/LITERAL_VALUES.md b/LITERAL_VALUES.md new file mode 100644 index 0000000..fccd72e --- /dev/null +++ b/LITERAL_VALUES.md @@ -0,0 +1,342 @@ +# Template Strategy: Literal Values + +## Overview + +Templates support static/literal values using the `lit:` prefix. This allows you to specify fallback values when other strategies don't produce results. + +## Syntax + +```json +{ + "field": "strategy1|strategy2|lit:static value" +} +``` + +## Strategy Types + +1. **heur** - Use heuristics (git history, document analysis) +2. **orig** - Use original frontmatter value +3. **ai ** - Use LLM inference +4. **lit:value** - Use literal/static value + +## Execution Order + +Strategies are tried **left to right** until one returns a non-null value. + +## Examples + +### Example 1: Author Fallback + +If git history has no authors, use a default: + +```json +{ + "authors": "heur|orig|lit:Ole Valente " +} +``` + +**Behavior:** +- Try heuristics (git authors) +- If empty, try original frontmatter +- If still empty, use "Ole Valente " + +### Example 2: Status Field + +Always set a status field to "draft": + +```json +{ + "status": "lit:draft" +} +``` + +**Behavior:** +- Immediately use "draft" (no other strategies) + +### Example 3: Version with Fallback + +```json +{ + "version": "heur|orig|lit:0.1.0" +} +``` + +**Behavior:** +- Try git tag/hash +- If no git, try original +- If still nothing, use "0.1.0" + +### Example 4: Category + +```json +{ + "category": "orig|lit:uncategorized" +} +``` + +**Behavior:** +- Use original category if exists +- Otherwise mark as "uncategorized" + +### Example 5: Complete Template with Fallbacks + +```json +{ + "title": "heur|orig|lit:Untitled Document", + "created": "heur|orig", + "changed": "heur|orig", + "authors": "heur|orig|lit:Unknown Author", + "version": "heur|orig|lit:0.1.0", + "tags": "heur|orig|ai default", + "status": "lit:draft", + "category": "orig|lit:general" +} +``` + +## List Values + +For list fields (like tags), use JSON array notation: + +```json +{ + "tags": "heur|orig|lit:[]" +} +``` + +This creates an empty list if no tags are found. + +Or with default tags: + +```json +{ + "keywords": "orig|lit:[\"general\", \"misc\"]" +} +``` + +**Note:** List values in `lit:` are parsed as strings, so complex structures should be avoided. For lists, prefer empty `[]` or use AI inference. + +## Common Patterns + +### Pattern 1: Required Field with Default + +```json +{ + "author": "orig|lit:Anonymous" +} +``` + +### Pattern 2: Preserve Original or Set Default + +```json +{ + "priority": "orig|lit:medium" +} +``` + +### Pattern 3: Try Everything, Then Default + +```json +{ + "summary": "orig|ai default|lit:No description available" +} +``` + +### Pattern 4: Static Value (No Fallback Needed) + +```json +{ + "type": "lit:document", + "format": "lit:markdown" +} +``` + +## Use Cases + +### 1. Personal Knowledge Base + +Set yourself as default author: + +```json +{ + "authors": "heur|lit:Jane Doe " +} +``` + +### 2. Organization Defaults + +```json +{ + "organization": "lit:Acme Corp", + "license": "lit:MIT", + "confidential": "lit:false" +} +``` + +### 3. Workflow States + +```json +{ + "status": "orig|lit:draft", + "reviewed": "orig|lit:false", + "published": "orig|lit:false" +} +``` + +### 4. Document Classification + +```json +{ + "type": "lit:article", + "category": "orig|ai default|lit:uncategorized", + "audience": "orig|lit:internal" +} +``` + +## Special Considerations + +### Empty Strings + +```json +{ + "note": "lit:" +} +``` + +This sets the field to an empty string. + +### Spaces in Values + +Spaces are preserved: + +```json +{ + "title": "lit:My Document Title" +} +``` + +Results in: `title: "My Document Title"` + +### Multiple Words + +No quotes needed in template: + +```json +{ + "author": "lit:John Smith" +} +``` + +### Email Addresses + +Special characters work fine: + +```json +{ + "contact": "lit:admin@example.com" +} +``` + +### Lists (Advanced) + +For simple lists, you can use: + +```json +{ + "reviewers": "lit:[]" +} +``` + +But for populated lists, prefer AI or heuristics as literal list parsing is string-based. + +## Testing Your Template + +### Test with --whatif + +```bash +madomeda --template my-template --whatif +``` + +### Check a specific file + +```bash +# Process just one file +madomeda --template my-template | grep -A 10 "Processing: myfile.md" +``` + +## Troubleshooting + +### Literal Value Not Applied + +**Issue:** Field is empty even with `lit:value` + +**Cause:** Another strategy succeeded earlier in the chain + +**Solution:** Move `lit:` to the beginning if you want it to always apply: +```json +{ + "status": "lit:draft" // Always use draft +} +``` + +### Wrong Value Type + +**Issue:** List field gets string instead of array + +**Cause:** `lit:` with complex JSON isn't parsed + +**Solution:** Use `lit:[]` for empty lists, or use AI/heuristics for populated lists + +## Complete Example Template + +**File:** `~/.config/madomeda/templates/blog-post.json` + +```json +{ + "title": "heur|orig|lit:Untitled Post", + "date": "heur|orig", + "author": "heur|orig|lit:Blog Team", + "category": "orig|lit:general", + "tags": "orig|ai default", + "status": "lit:draft", + "published": "orig|lit:false", + "featured": "orig|lit:false", + "excerpt": "orig|ai excerpt" +} +``` + +**Usage:** +```bash +madomeda --template blog-post +``` + +**Result for new file:** +```yaml +--- +title: My Blog Post # From # heading +date: '2025-10-23T19:30:00+02:00' # From git +author: Blog Team # Literal fallback (no git author found) +category: general # Literal (no original) +tags: # From AI +- blogging +- tutorials +status: draft # Literal +published: false # Literal +featured: false # Literal +excerpt: A comprehensive guide... # From AI +--- +``` + +## Best Practices + +1. **Always provide fallbacks** for critical fields +2. **Use heur first** to leverage git metadata +3. **Use orig second** to preserve existing values +4. **Use lit last** as safety net +5. **Keep literals simple** - avoid complex structures +6. **Document your templates** in comments (though JSON doesn't support them, keep external docs) + +## See Also + +- [CONFIGURATION.md](CONFIGURATION.md) - Template configuration +- [USAGE.md](USAGE.md) - Template usage examples +- [STRUCTURE.md](STRUCTURE.md) - Template system architecture diff --git a/PROJECT_SUMMARY.md b/PROJECT_SUMMARY.md index 089a075..8e06a0d 100644 --- a/PROJECT_SUMMARY.md +++ b/PROJECT_SUMMARY.md @@ -1,11 +1,11 @@ --- title: Madomeda - Project Complete created: '2025-10-23T18:33:48+02:00' -changed: '2025-10-23T18:33:48+02:00' -authors: -- Test User -version: 8c4f612 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Madomeda - Project Complete diff --git a/README.md b/README.md index b9617bc..ac4f92c 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ --- title: Madomeda - Markdown Document Metadata Manager created: '2025-10-23T18:33:48+02:00' -changed: '2025-10-23T18:33:48+02:00' -authors: -- Test User -version: 8c4f612 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Madomeda - Markdown Document Metadata Manager @@ -59,6 +59,7 @@ Multi-strategy field resolution: - `heur`: Heuristics (git history, document content) - `orig`: Original frontmatter value - `ai `: LLM inference with structured output +- `lit:value`: Literal/static value (fallback) ### Database Tracking SQLite database stores: @@ -151,6 +152,9 @@ tags: - **[GETTING_STARTED.md](GETTING_STARTED.md)** - Step-by-step guide - **[USAGE.md](USAGE.md)** - Detailed usage examples - **[CONFIGURATION.md](CONFIGURATION.md)** - Configuration management +- **[RULES_GUIDE.md](RULES_GUIDE.md)** - Writing custom rules +- **[TEMPLATE_TYPES.md](TEMPLATE_TYPES.md)** - Template field types +- **[LITERAL_VALUES.md](LITERAL_VALUES.md)** - Template literal values - **[STRUCTURE.md](STRUCTURE.md)** - Project architecture - **[IMPLEMENTATION.md](IMPLEMENTATION.md)** - Technical details - **[PROJECT_SUMMARY.md](PROJECT_SUMMARY.md)** - Complete overview @@ -187,21 +191,25 @@ madomeda/ ## Templates -Create custom templates in `templates/`: +Create custom templates in `~/.config/madomeda/templates/`: ```json { - "title": "heur|orig", - "created": "heur|orig", - "changed": "heur|orig", - "authors": "heur|orig", - "version": "heur|orig", - "tags": "heur|orig|ai default" + "title": "heur|orig:text", + "created": "heur|orig:datetime", + "changed": "heur|orig:datetime", + "authors": "heur|orig:list", + "version": "heur|orig:text", + "tags": "heur|orig|ai default:list", + "status": "lit:draft:text", + "published": "orig|lit:false:checkbox" } ``` **Strategy priority**: Left to right until one succeeds. +**Field types**: `text`, `number`, `checkbox`, `date`, `datetime`, `list` (see [TEMPLATE_TYPES.md](TEMPLATE_TYPES.md)) + ## Rules Default rules in `rules/`: diff --git a/RULES_GUIDE.md b/RULES_GUIDE.md new file mode 100644 index 0000000..052d333 --- /dev/null +++ b/RULES_GUIDE.md @@ -0,0 +1,562 @@ +# Writing Custom Rules + +## Overview + +Madomeda uses a self-contained, JSON-based rules system. Rules are processed in priority order and can normalize, validate, and transform frontmatter fields without any code changes. + +## Rule File Location + +Rules are stored in: `~/.config/madomeda/rules/` + +Each rule is a separate JSON file. The filename is used for sorting (alphabetically), but the `priority` field determines execution order. + +## Rule Structure + +```json +{ + "name": "rule_name", + "description": "Human-readable description", + "field": "field_name", + "priority": 10, + "action": "action_type", + "pattern": "regex_pattern", + "replacement": "replacement_string", + "llm_prompt": "Prompt for LLM if heuristics fail" +} +``` + +### Required Fields + +- **name**: Unique identifier for the rule +- **description**: What the rule does +- **field**: Which frontmatter field this applies to (use `"*"` for all fields) +- **priority**: Execution order (lower number = earlier execution, typically 1-100) +- **action**: What the rule does (see Actions below) + +### Optional Fields + +- **pattern**: Regex pattern for matching/replacing +- **replacement**: Replacement string (can use capture groups like `\g<0>`, `\1`, etc.) +- **transform**: Transformation to apply (`lower`, `upper`) +- **multiline**: Whether regex uses multiline mode (default: false) +- **split_on**: Character to split on (e.g., `","` for comma-separated values) +- **from**: Source field name (for rename actions) +- **to**: Target field name (for rename actions) +- **llm_prompt**: Text to send to LLM if this rule's value needs inference + +## Actions + +### 1. normalize_keys + +Apply transformation to all frontmatter keys. + +**Use case**: Ensure all keys are lowercase. + +**Example:** +```json +{ + "name": "lowercase_keys", + "description": "Convert all keys to lowercase", + "field": "*", + "priority": 1, + "action": "normalize_keys", + "transform": "lower", + "llm_prompt": "Ensure all keys are lowercase" +} +``` + +**Alternative with regex:** +```json +{ + "name": "remove_spaces_from_keys", + "description": "Remove spaces from keys", + "field": "*", + "priority": 2, + "action": "normalize_keys", + "pattern": " ", + "replacement": "_" +} +``` + +### 2. rename_field + +Rename one field to another. + +**Use case**: Standardize field names (e.g., `tag` → `tags`, `summary` → `description`). + +**Example:** +```json +{ + "name": "tag_to_tags", + "description": "Rename 'tag' to 'tags'", + "field": "tag", + "priority": 2, + "action": "rename_field", + "from": "tag", + "to": "tags", + "llm_prompt": "Convert tag field to tags array" +} +``` + +**Note**: Rename rules execute before other field-specific rules, so subsequent rules can process the renamed field. + +### 3. normalize_value + +Transform field values using regex, transforms, or splitting. + +**Use case**: Standardize tag format, clean up values, convert formats. + +**Example 1: Split comma-separated to list** +```json +{ + "name": "tags_split", + "description": "Convert comma-separated tags to list", + "field": "tags", + "priority": 10, + "action": "normalize_value", + "split_on": ",", + "pattern": ".*", + "replacement": "\\g<0>" +} +``` + +**Example 2: Replace spaces with underscores** +```json +{ + "name": "tags_no_spaces", + "description": "Replace spaces with underscores in tags", + "field": "tags", + "priority": 20, + "action": "normalize_value", + "pattern": " ", + "replacement": "_" +} +``` + +**Example 3: Lowercase transformation** +```json +{ + "name": "tags_lowercase", + "description": "Convert tags to lowercase", + "field": "tags", + "priority": 30, + "action": "normalize_value", + "transform": "lower" +} +``` + +**Example 4: Remove invalid characters** +```json +{ + "name": "tags_alphanumeric_only", + "description": "Keep only alphanumeric and underscores", + "field": "tags", + "priority": 40, + "action": "normalize_value", + "pattern": "[^a-z0-9_]", + "replacement": "" +} +``` + +### 4. validate + +Check if field values match a pattern. + +**Use case**: Ensure values conform to expected format. + +**Example:** +```json +{ + "name": "tags_validate", + "description": "Validate tag format", + "field": "tags", + "priority": 50, + "action": "validate", + "pattern": "^[a-z0-9_]+$", + "multiline": false +} +``` + +**Note**: Validation rules mark frontmatter as non-conformant if they fail, but don't modify values. + +## Priority System + +Rules execute in priority order (lowest number first): + +- **1-9**: Global transformations (keys, field renames) +- **10-19**: Format conversions (split lists, type changes) +- **20-29**: Character replacements (spaces, hyphens) +- **30-39**: Case transformations +- **40-49**: Character removal (invalid chars) +- **50-99**: Validation + +**Recommended naming convention:** +``` +01_lowercase_keys.json # Priority 1 +10_tags_format_list.json # Priority 10 +20_tags_replace_spaces.json # Priority 20 +50_tags_validate.json # Priority 50 +``` + +## Processing Flow + +1. **Global rules** (`field: "*"`) execute first +2. **Rename rules** execute second (creates new fields) +3. **Other rules** execute in priority order per field + +## Complete Example: Tag Normalization Chain + +Here's how multiple rules work together to normalize tags: + +**Input:** +```yaml +Tag: Machine-Learning, Deep Learning, AI/ML +``` + +**Rules (in priority order):** + +```json +// 02_tag_to_tags.json +{ + "name": "tag_to_tags", + "field": "tag", + "priority": 2, + "action": "rename_field", + "from": "tag", + "to": "tags" +} +``` + +After this rule: `tags: "Machine-Learning, Deep Learning, AI/ML"` + +```json +// 10_tags_format_list.json +{ + "name": "tags_split", + "field": "tags", + "priority": 10, + "action": "normalize_value", + "split_on": "," +} +``` + +After this rule: `tags: ["Machine-Learning", "Deep Learning", "AI/ML"]` + +```json +// 20_tags_replace_spaces.json +{ + "name": "tags_spaces", + "field": "tags", + "priority": 20, + "action": "normalize_value", + "pattern": " ", + "replacement": "_" +} +``` + +After this rule: `tags: ["Machine-Learning", "Deep_Learning", "AI/ML"]` + +```json +// 21_tags_replace_hyphens.json +{ + "name": "tags_hyphens", + "field": "tags", + "priority": 21, + "action": "normalize_value", + "pattern": "-", + "replacement": "_" +} +``` + +After this rule: `tags: ["Machine_Learning", "Deep_Learning", "AI/ML"]` + +```json +// 30_tags_lowercase.json +{ + "name": "tags_lowercase", + "field": "tags", + "priority": 30, + "action": "normalize_value", + "transform": "lower" +} +``` + +After this rule: `tags: ["machine_learning", "deep_learning", "ai/ml"]` + +```json +// 40_tags_remove_invalid.json +{ + "name": "tags_clean", + "field": "tags", + "priority": 40, + "action": "normalize_value", + "pattern": "[^a-z0-9_]", + "replacement": "" +} +``` + +**Final result:** `tags: ["machine_learning", "deep_learning", "aiml"]` + +## Field Types + +Rules apply to different value types: + +### String Fields + +Rules apply to the whole string: + +```json +{ + "field": "title", + "action": "normalize_value", + "transform": "lower" +} +``` + +### List Fields + +Rules apply to each item in the list: + +```json +{ + "field": "keywords", + "action": "normalize_value", + "pattern": " ", + "replacement": "_" +} +``` + +Each keyword gets spaces replaced with underscores. + +### Converting String to List + +Use `split_on`: + +```json +{ + "field": "authors", + "action": "normalize_value", + "split_on": ";", + "pattern": ".*", + "replacement": "\\g<0>" +} +``` + +Input: `"John Doe; Jane Smith"` +Output: `["John Doe", "Jane Smith"]` + +## Regular Expression Tips + +### Capture Groups + +```json +{ + "pattern": "(\\d+)", + "replacement": "v\\1" +} +``` + +Input: `"123"` +Output: `"v123"` + +### Case-Insensitive Matching + +Use inline flag: + +```json +{ + "pattern": "(?i)todo", + "replacement": "TODO" +} +``` + +### Match Whole String + +```json +{ + "pattern": "^[a-z]+$" +} +``` + +### Remove Leading/Trailing Whitespace + +```json +{ + "pattern": "^\\s+|\\s+$", + "replacement": "" +} +``` + +## LLM Integration + +The `llm_prompt` field provides guidance when the LLM needs to generate or fix values: + +```json +{ + "name": "tags_validate", + "field": "tags", + "action": "validate", + "pattern": "^[a-z0-9_]+$", + "llm_prompt": "Generate tags using only lowercase letters, numbers, and underscores. Prefer selecting from existing repository tags when appropriate." +} +``` + +When using `ai` strategy in templates, the LLM receives: +- The rule's `llm_prompt` +- List of existing tags in repository +- Document content +- Structured output schema + +## Testing Rules + +### Test with --whatif + +```bash +madomeda --whatif +``` + +Shows what would change without modifying files. + +### Test Specific Rule + +Create a Python script: + +```python +from core.rules_processor import RulesProcessor +from pathlib import Path + +rp = RulesProcessor(Path.home() / '.config' / 'madomeda' / 'rules') + +test_data = { + 'Title': 'My Document', + 'Tag': 'Python, Machine-Learning' +} + +result, conformant, violations = rp.apply_rules(test_data) +print(f"Result: {result}") +print(f"Conformant: {conformant}") +print(f"Violations: {violations}") +``` + +## Common Patterns + +### Email Validation + +```json +{ + "field": "author_email", + "action": "validate", + "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$" +} +``` + +### Date Format Validation (YYYY-MM-DD) + +```json +{ + "field": "date", + "action": "validate", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$" +} +``` + +### URL Scheme Enforcement + +```json +{ + "field": "url", + "action": "normalize_value", + "pattern": "^(?!https?://)(.*)", + "replacement": "https://\\1" +} +``` + +### Remove HTML Tags + +```json +{ + "field": "description", + "action": "normalize_value", + "pattern": "<[^>]+>", + "replacement": "" +} +``` + +## Advanced: Custom Field Rules + +### Ensure Boolean String Format + +```json +{ + "field": "published", + "action": "normalize_value", + "pattern": "^(true|false|yes|no|1|0)$", + "replacement": "\\1", + "transform": "lower" +} +``` + +Then normalize: + +```json +{ + "field": "published", + "priority": 31, + "action": "normalize_value", + "pattern": "yes|1|true", + "replacement": "true" +} +``` + +### Slug Generation + +```json +{ + "field": "slug", + "action": "normalize_value", + "pattern": "[^a-z0-9-]", + "replacement": "", + "transform": "lower" +} +``` + +## Troubleshooting + +### Rule Not Executing + +- Check `field` matches exact field name +- Verify `priority` is in expected range +- Ensure JSON is valid (use `jsonlint`) + +### Wrong Execution Order + +- Lower priority number executes first +- Rename rules always execute before other rules for same field +- Global rules (`field: "*"`) execute before field-specific + +### Regex Not Matching + +- Test regex at https://regex101.com/ +- Escape special characters: `. * + ? ^ $ { } ( ) | [ ] \` +- Use `multiline: true` for multiline patterns + +### Value Not Changing + +- Check if earlier rule already modified it +- Verify pattern actually matches the value +- Ensure `replacement` is specified for `normalize_value` + +## Best Practices + +1. **Use priority ranges** - Leave gaps (10, 20, 30) to insert rules later +2. **Name files by priority** - `01_`, `10_`, `20_` for easy sorting +3. **Test incrementally** - Add one rule at a time +4. **Document complex regex** - Use `description` field +5. **Provide LLM prompts** - Help AI understand the rule intent +6. **Validate after normalize** - Use separate validate rule at higher priority + +## See Also + +- [CONFIGURATION.md](CONFIGURATION.md) - Configuration directory +- [STRUCTURE.md](STRUCTURE.md) - How rules fit into the system +- [USAGE.md](USAGE.md) - Using rules with madomeda diff --git a/STRUCTURE.md b/STRUCTURE.md index 181d37e..03f578e 100644 --- a/STRUCTURE.md +++ b/STRUCTURE.md @@ -1,11 +1,11 @@ --- title: Madomeda Project Structure created: '2025-10-23T18:33:48+02:00' -changed: '2025-10-23T18:33:48+02:00' -authors: -- Test User -version: 8c4f612 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Madomeda Project Structure diff --git a/TEMPLATE_TYPES.md b/TEMPLATE_TYPES.md new file mode 100644 index 0000000..6de7375 --- /dev/null +++ b/TEMPLATE_TYPES.md @@ -0,0 +1,470 @@ +# Template Field Types + +## Overview + +Templates support explicit type declarations for frontmatter fields. This ensures correct formatting and enables proper LLM schema generation. + +## Syntax + +```json +{ + "fieldname": "strategy|strategy:type" +} +``` + +The type is specified after a colon `:` at the end of the strategy chain. + +## Supported Types + +| Type | Description | YAML Output | Example | +|------|-------------|-------------|---------| +| `text` | String value (default) | `field: value` | Title, description | +| `number` | Integer or float | `field: 42` | Count, score | +| `checkbox` | Boolean value | `field: true` | Published, active | +| `date` | Date only (YAML ISO date) | `field: '2025-10-23'` | Birth date, deadline | +| `datetime` | Date and time (YAML ISO 8601) | `field: '2025-10-23T19:54:00+02:00'` | Created, modified | +| `list` | Array of strings | `field: [item1, item2]` | Tags, keywords, authors | + +If no type is specified, `text` is assumed. + +## Examples + +### Basic Template with Types + +```json +{ + "title": "heur|orig:text", + "created": "heur|orig:datetime", + "changed": "heur|orig:datetime", + "authors": "heur|orig:list", + "version": "heur|orig:text", + "tags": "heur|orig|ai default:list", + "published": "orig|lit:false:checkbox", + "word_count": "heur:number" +} +``` + +### Text Fields + +**Template:** +```json +{ + "title": "heur|orig:text", + "summary": "orig|ai default:text", + "author_name": "heur|lit:Anonymous:text" +} +``` + +**Output:** +```yaml +--- +title: My Document +summary: This is a summary +author_name: Anonymous +--- +``` + +### Number Fields + +**Template:** +```json +{ + "priority": "orig|lit:5:number", + "version_number": "heur:number", + "word_count": "heur:number" +} +``` + +**Output:** +```yaml +--- +priority: 5 +version_number: 2 +word_count: 1532 +--- +``` + +**Literal conversion:** +- `lit:42:number` → `42` (integer) +- `lit:3.14:number` → `3.14` (float) +- Invalid numbers default to string + +### Checkbox Fields + +**Template:** +```json +{ + "published": "orig|lit:false:checkbox", + "draft": "lit:true:checkbox", + "featured": "orig|lit:false:checkbox" +} +``` + +**Output:** +```yaml +--- +published: false +draft: true +featured: false +--- +``` + +**Literal conversion:** +- `true`, `1`, `yes`, `on` → `true` +- Anything else → `false` + +### Date Fields + +**Template:** +```json +{ + "birth_date": "orig:date", + "deadline": "heur:date" +} +``` + +**Output:** +```yaml +--- +birth_date: '2025-10-23' +deadline: '2025-12-31' +--- +``` + +**Format:** ISO 8601 date (`YYYY-MM-DD`) + +### Datetime Fields + +**Template:** +```json +{ + "created": "heur:datetime", + "modified": "heur:datetime", + "published_at": "orig:datetime" +} +``` + +**Output:** +```yaml +--- +created: '2025-10-23T19:54:00+02:00' +modified: '2025-10-23T20:15:30+02:00' +published_at: '2025-10-23T18:00:00+00:00' +--- +``` + +**Format:** ISO 8601 with timezone + +### List Fields + +**Template:** +```json +{ + "tags": "heur|orig|ai default:list", + "authors": "heur|orig:list", + "keywords": "orig|lit:[]:list", + "categories": "ai categorize:list" +} +``` + +**Output:** +```yaml +--- +tags: +- machine_learning +- python +- tutorial +authors: +- John Doe +- Jane Smith +keywords: [] +categories: +- technology +- programming +--- +``` + +**Literal list:** +```json +{ + "reviewers": "lit:[]:list" +} +``` + +Creates empty list: `reviewers: []` + +## Type with Strategies + +### Literal Values + +```json +{ + "status": "lit:draft:text", + "priority": "lit:3:number", + "urgent": "lit:false:checkbox", + "default_tags": "lit:[]:list" +} +``` + +### Heuristic Values + +Heuristics automatically infer appropriate types: + +```json +{ + "created": "heur:datetime", // From git (ISO 8601) + "changed": "heur:datetime", // From git (ISO 8601) + "authors": "heur:list", // From git (list of names) + "version": "heur:text", // From git (tag or hash) + "title": "heur:text", // From # heading + "tags": "heur:list" // From normalized tags +} +``` + +### Original Values + +Preserves the original type if it matches: + +```json +{ + "tags": "orig:list", // Keep original tags list + "published": "orig:checkbox", // Keep original boolean + "score": "orig:number" // Keep original number +} +``` + +### AI Values + +LLM generates values conforming to the specified type: + +```json +{ + "summary": "ai summarize:text", + "tags": "ai default:list", + "relevance_score": "ai evaluate:number", + "needs_review": "ai check:checkbox" +} +``` + +The type determines the JSON schema sent to the LLM. + +## Complex Templates + +### Blog Post + +```json +{ + "title": "heur|orig:text", + "date": "heur:datetime", + "author": "heur|lit:Blog Team:text", + "category": "orig|ai categorize:text", + "tags": "orig|ai default:list", + "published": "orig|lit:false:checkbox", + "featured": "orig|lit:false:checkbox", + "word_count": "heur:number", + "reading_time": "heur:number", + "excerpt": "orig|ai excerpt:text" +} +``` + +### Documentation + +```json +{ + "title": "heur|orig:text", + "version": "heur:text", + "created": "heur:datetime", + "updated": "heur:datetime", + "authors": "heur:list", + "reviewers": "orig|lit:[]:list", + "status": "orig|lit:draft:text", + "tags": "heur|orig|ai default:list", + "api_version": "orig:text", + "deprecated": "orig|lit:false:checkbox" +} +``` + +### Task/Todo + +```json +{ + "title": "heur|orig:text", + "created": "heur:datetime", + "due_date": "orig:date", + "priority": "orig|lit:3:number", + "completed": "orig|lit:false:checkbox", + "assigned_to": "orig|lit:[]:list", + "tags": "orig:list", + "estimated_hours": "orig:number" +} +``` + +## Type Validation + +### Automatic Conversion + +When using literal values, types are automatically converted: + +```json +{ + "published": "lit:true:checkbox" // String "true" → boolean true +} +``` + +### Invalid Values + +If conversion fails, value is kept as string: + +```json +{ + "count": "lit:invalid:number" // "invalid" → "invalid" (string) +} +``` + +## LLM Schema Generation + +Types control the JSON schema sent to LLMs: + +**Template:** +```json +{ + "title": "ai generate:text", + "tags": "ai generate:list", + "relevance": "ai score:number", + "approved": "ai check:checkbox" +} +``` + +**Generated Schema:** +```json +{ + "type": "object", + "properties": { + "title": {"type": "string"}, + "tags": {"type": "array", "items": {"type": "string"}}, + "relevance": {"type": "number"}, + "approved": {"type": "boolean"} + } +} +``` + +This ensures LLM returns correctly typed values. + +## Best Practices + +### 1. Always Specify Types for Lists + +```json +{ + "tags": "heur|orig:list", // ✅ Explicit + "tags": "heur|orig" // ❌ Ambiguous +} +``` + +### 2. Use Checkbox for Booleans + +```json +{ + "published": "orig|lit:false:checkbox", // ✅ Boolean + "published": "orig|lit:no:text" // ❌ String +} +``` + +### 3. Datetime for Timestamps + +```json +{ + "created": "heur:datetime", // ✅ Full timestamp + "created": "heur:text" // ❌ Generic +} +``` + +### 4. Number for Numeric Data + +```json +{ + "priority": "orig|lit:5:number", // ✅ Number + "priority": "orig|lit:5:text" // ❌ String "5" +} +``` + +### 5. List Default Values + +```json +{ + "keywords": "orig|lit:[]:list", // ✅ Empty list + "keywords": "orig:list" // ⚠️ May be null +} +``` + +## Migration from Untyped Templates + +### Before (Untyped) + +```json +{ + "title": "heur|orig", + "tags": "heur|orig|ai default", + "published": "lit:false" +} +``` + +### After (Typed) + +```json +{ + "title": "heur|orig:text", + "tags": "heur|orig|ai default:list", + "published": "lit:false:checkbox" +} +``` + +**Note:** Untyped templates still work (default to `text`), but typed templates are recommended for clarity and correctness. + +## Troubleshooting + +### List Shows as String + +**Problem:** `tags: "tag1, tag2"` instead of `tags: [tag1, tag2]` + +**Solution:** Specify `:list` type and ensure rules split comma-separated values: +```json +{ + "tags": "heur|orig:list" +} +``` + +### Boolean Shows as String + +**Problem:** `published: "true"` instead of `published: true` + +**Solution:** Use `:checkbox` type: +```json +{ + "published": "orig|lit:false:checkbox" +} +``` + +### Number Shows as String + +**Problem:** `priority: "5"` instead of `priority: 5` + +**Solution:** Use `:number` type: +```json +{ + "priority": "orig|lit:5:number" +} +``` + +### Date Format Wrong + +**Problem:** `date: "10/23/2025"` instead of `date: '2025-10-23'` + +**Solution:** Use `:date` or `:datetime` type and ensure source provides ISO 8601 format. + +## See Also + +- [LITERAL_VALUES.md](LITERAL_VALUES.md) - Literal value syntax +- [CONFIGURATION.md](CONFIGURATION.md) - Template configuration +- [USAGE.md](USAGE.md) - Template usage examples +- [RULES_GUIDE.md](RULES_GUIDE.md) - Rule writing guide diff --git a/USAGE.md b/USAGE.md index 3358f9a..8cdfd3d 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1,11 +1,11 @@ --- title: Madomeda Usage Examples created: '2025-10-23T18:33:48+02:00' -changed: '2025-10-23T18:33:48+02:00' -authors: -- Test User -version: 8c4f612 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Madomeda Usage Examples diff --git a/VERSION_FIELD.md b/VERSION_FIELD.md index ec21d4e..3981edc 100644 --- a/VERSION_FIELD.md +++ b/VERSION_FIELD.md @@ -1,11 +1,11 @@ --- title: Version Field Behavior created: '2025-10-23T18:33:48+02:00' -changed: '2025-10-23T18:33:48+02:00' -authors: -- Test User -version: 8c4f612 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Version Field Behavior diff --git a/advanced.md b/advanced.md index ec2bea4..c4cb56a 100644 --- a/advanced.md +++ b/advanced.md @@ -1,14 +1,14 @@ --- title: Advanced Machine Learning Techniques created: '2025-10-23T17:49:29+02:00' -changed: '2025-10-23T17:49:29+02:00' -authors: -- Test User -version: 2e822d8 +status: draft +priority: 5 +published: false tags: - aiml - deep_learning - neural_networks +reviewers: [] --- # Advanced Machine Learning Techniques diff --git a/ai_test.md b/ai_test.md index 9654656..81c1a5b 100644 --- a/ai_test.md +++ b/ai_test.md @@ -1,16 +1,16 @@ --- title: AI Testing Document created: '2025-10-23T19:22:20+02:00' -changed: '2025-10-23T19:22:20+02:00' -authors: -- Test User -version: ab4b95c +status: draft +priority: 5 +published: false tags: - aiml - data_science - deep_learning - machine_learning - neural_networks +reviewers: [] --- # AI Testing Document diff --git a/comparison.md b/comparison.md index fa46a3d..d197dd7 100644 --- a/comparison.md +++ b/comparison.md @@ -1,11 +1,11 @@ --- title: Comparison Test created: '2025-10-23T18:09:10+02:00' -changed: '2025-10-23T18:09:10+02:00' -authors: -- Test User -version: v1.0.0 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Comparison Test diff --git a/core/config.py b/core/config.py index e4767af..e65aced 100644 --- a/core/config.py +++ b/core/config.py @@ -76,12 +76,12 @@ OPENAI_API_KEY="" """Create default template""" import json default_template = { - "title": "heur|orig", - "created": "heur|orig", - "changed": "heur|orig", - "authors": "heur|orig", - "version": "heur|orig", - "tags": "heur|orig|ai default" + "title": "heur|orig:text", + "created": "heur|orig:datetime", + "changed": "heur|orig:datetime", + "authors": "heur|orig:list", + "version": "heur|orig:text", + "tags": "heur|orig|ai default:list" } with open(self.templates_dir / 'default.json', 'w', encoding='utf-8') as f: json.dump(default_template, f, indent=2) @@ -91,25 +91,94 @@ OPENAI_API_KEY="" import json rules = { - 'tags_normalization.json': { - "name": "tags_normalization", - "description": "Normalize tags to lowercase with underscores", - "type": "tags" - }, - 'tag_to_tags.json': { - "name": "tag_to_tags", - "description": "Rename 'tag' key to 'tags'", - "type": "rename" - }, - 'lowercase_keys.json': { + '01_lowercase_keys.json': { "name": "lowercase_keys", "description": "All frontmatter keys must be lowercase", - "type": "keys" + "field": "*", + "priority": 1, + "action": "normalize_keys", + "transform": "lower", + "llm_prompt": "Ensure all keys are lowercase" }, - 'summary_to_description.json': { + '02_tag_to_tags.json': { + "name": "tag_to_tags", + "description": "Rename 'tag' field to 'tags'", + "field": "tag", + "priority": 2, + "action": "rename_field", + "from": "tag", + "to": "tags", + "llm_prompt": "Convert tag field to tags array" + }, + '03_summary_to_description.json': { "name": "summary_to_description", - "description": "Rename 'summary' key to 'description'", - "type": "rename" + "description": "Rename 'summary' field to 'description'", + "field": "summary", + "priority": 2, + "action": "rename_field", + "from": "summary", + "to": "description", + "llm_prompt": "Convert summary to description" + }, + '10_tags_format_list.json': { + "name": "tags_format_list", + "description": "Convert comma-separated tags to list", + "field": "tags", + "priority": 10, + "action": "normalize_value", + "split_on": ",", + "pattern": ".*", + "replacement": "\\g<0>", + "llm_prompt": "Ensure tags is a list, not a comma-separated string" + }, + '20_tags_replace_spaces.json': { + "name": "tags_replace_spaces", + "description": "Replace spaces with underscores in tags", + "field": "tags", + "priority": 20, + "action": "normalize_value", + "pattern": " ", + "replacement": "_", + "llm_prompt": "Replace spaces with underscores in tags" + }, + '21_tags_replace_hyphens.json': { + "name": "tags_replace_hyphens", + "description": "Replace hyphens with underscores in tags", + "field": "tags", + "priority": 21, + "action": "normalize_value", + "pattern": "-", + "replacement": "_", + "llm_prompt": "Replace hyphens with underscores in tags" + }, + '30_tags_lowercase.json': { + "name": "tags_lowercase", + "description": "Convert tags to lowercase", + "field": "tags", + "priority": 30, + "action": "normalize_value", + "transform": "lower", + "llm_prompt": "Convert all tags to lowercase" + }, + '40_tags_remove_invalid_chars.json': { + "name": "tags_remove_invalid_chars", + "description": "Remove characters that are not alphanumeric or underscore", + "field": "tags", + "priority": 40, + "action": "normalize_value", + "pattern": "[^a-z0-9_]", + "replacement": "", + "llm_prompt": "Remove any characters that are not lowercase letters, numbers, or underscores from tags" + }, + '50_tags_validate_format.json': { + "name": "tags_validate_format", + "description": "Validate that tags only contain lowercase alphanumeric and underscores", + "field": "tags", + "priority": 50, + "action": "validate", + "pattern": "^[a-z0-9_]+$", + "multiline": False, + "llm_prompt": "Tags must contain only lowercase letters, numbers, and underscores" } } diff --git a/core/processor.py b/core/processor.py index b056a0c..7cca888 100644 --- a/core/processor.py +++ b/core/processor.py @@ -7,7 +7,7 @@ from datetime import datetime from core.database import Database from core.repository import Repository from core.frontmatter import FrontmatterParser -from core.rules import RulesEngine +from core.rules_processor import RulesProcessor from core.template import TemplateManager from core.llm import LLMClient from core.changelog import ChangeLog @@ -32,7 +32,7 @@ class FrontmatterProcessor: api_config = self.config.get_api_config() self.db = Database(repo.path / 'madomeda.db') - self.rules = RulesEngine(self.config.rules_dir) + self.rules = RulesProcessor(self.config.rules_dir) self.templates = TemplateManager(self.config.templates_dir) self.llm = LLMClient(self.config.prompts_dir, api_config) self.changelog = ChangeLog(repo.path) @@ -113,7 +113,7 @@ class FrontmatterProcessor: ) # Apply rules - normalized_fm, is_conformant = self.rules.apply_rules(dict(original_fm)) + normalized_fm, is_conformant, violations = self.rules.apply_rules(dict(original_fm)) # Check if we need to update needs_update = self.force or not is_conformant or not self._matches_template(normalized_fm, template) @@ -190,14 +190,21 @@ class FrontmatterProcessor: new_fm = {} all_tags = self.db.get_all_tags() - for field, strategy in template.items(): - options = strategy.split('|') + for field, field_value in template.items(): + # Parse field to get strategy and type + strategy_str, field_type = self.templates.parse_field(str(field_value)) + options = strategy_str.split('|') value = None for option in options: option = option.strip() - if option == 'heur': + if option.startswith('lit:'): + # Literal/static value + value = option[4:].strip() + # Convert based on type + value = self._convert_value_type(value, field_type) + elif option == 'heur': # Use heuristics value = self._get_heuristic_value(field, normalized_fm, body, file_path, git_info) elif option == 'orig': @@ -218,6 +225,30 @@ class FrontmatterProcessor: return new_fm + def _convert_value_type(self, value: str, field_type: str): + """Convert literal value to appropriate type""" + if field_type == 'checkbox': + return value.lower() in ('true', '1', 'yes', 'on') + elif field_type == 'number': + try: + if '.' in value: + return float(value) + return int(value) + except ValueError: + return value + elif field_type == 'list': + # Parse as JSON array or return empty list + if value == '[]': + return [] + try: + import json + return json.loads(value) + except: + return [value] + else: + # text, date, datetime - keep as string + return value + def _get_heuristic_value(self, field: str, normalized_fm: dict, body: str, file_path: Path, git_info: dict): """Get field value using heuristics""" diff --git a/core/rules_processor.py b/core/rules_processor.py new file mode 100644 index 0000000..7d597a9 --- /dev/null +++ b/core/rules_processor.py @@ -0,0 +1,255 @@ +""" +Self-contained rules processor +Processes JSON-based rules without hardcoded logic +""" +import json +import re +from pathlib import Path +from typing import List, Tuple, Dict, Any + + +class RulesProcessor: + def __init__(self, rules_dir: Path): + self.rules_dir = rules_dir + self.rules = self._load_rules() + + def _load_rules(self) -> Dict[str, List[dict]]: + """Load all rule files and organize by field""" + rules_by_field = {} + + if self.rules_dir.exists(): + for rule_file in self.rules_dir.glob('*.json'): + with open(rule_file, 'r', encoding='utf-8') as f: + rule = json.load(f) + field = rule.get('field', '*') + + if field not in rules_by_field: + rules_by_field[field] = [] + + rules_by_field[field].append(rule) + + # Sort rules by priority (lower number = higher priority) + for field in rules_by_field: + rules_by_field[field].sort(key=lambda r: r.get('priority', 100)) + + return rules_by_field + + def apply_rules(self, frontmatter: dict) -> Tuple[dict, bool, List[str]]: + """ + Apply all rules to frontmatter + Returns: (modified_fm, is_conformant, violations) + """ + if not frontmatter: + return frontmatter, True, [] + + modified = dict(frontmatter) + violations = [] + + # Process global rules (field: "*") first + if '*' in self.rules: + for rule in self.rules['*']: + modified, rule_violations = self._apply_rule(modified, rule, None) + violations.extend(rule_violations) + + # Process rename rules to create new fields + for field in list(modified.keys()): + if field in self.rules: + for rule in self.rules[field]: + if rule.get('action') == 'rename_field': + modified, rule_violations = self._apply_rule(modified, rule, field) + violations.extend(rule_violations) + + # Process other field-specific rules on current fields + for field in list(modified.keys()): + if field in self.rules: + for rule in self.rules[field]: + if rule.get('action') != 'rename_field': # Skip renames, already done + modified, rule_violations = self._apply_rule(modified, rule, field) + violations.extend(rule_violations) + + is_conformant = len(violations) == 0 + return modified, is_conformant, violations + + def _apply_rule(self, frontmatter: dict, rule: dict, field: str = None) -> Tuple[dict, List[str]]: + """Apply a single rule to frontmatter""" + modified = dict(frontmatter) + violations = [] + action = rule.get('action', 'validate') + + if action == 'normalize_keys': + modified, violated = self._action_normalize_keys(modified, rule) + if violated: + violations.append(rule.get('name', 'unknown')) + + elif action == 'rename_field': + modified, violated = self._action_rename_field(modified, rule) + if violated: + violations.append(rule.get('name', 'unknown')) + + elif action == 'normalize_value': + if field and field in modified: + modified[field], violated = self._action_normalize_value(modified[field], rule) + if violated: + violations.append(rule.get('name', 'unknown')) + + elif action == 'validate': + if field and field in modified: + violated = not self._action_validate(modified[field], rule) + if violated: + violations.append(rule.get('name', 'unknown')) + + return modified, violations + + def _action_normalize_keys(self, frontmatter: dict, rule: dict) -> Tuple[dict, bool]: + """Normalize all keys using regex pattern or transform""" + pattern = rule.get('pattern', '') + replacement = rule.get('replacement', '') + transform = rule.get('transform', '') + + modified = {} + violated = False + + for key, value in frontmatter.items(): + new_key = key + + # Apply transform (e.g., lower, upper) + if transform == 'lower': + new_key = key.lower() + elif transform == 'upper': + new_key = key.upper() + elif pattern: + # Apply regex pattern + new_key = re.sub(pattern, replacement, key, flags=re.MULTILINE if rule.get('multiline') else 0) + + if new_key != key: + violated = True + modified[new_key] = value + + return modified, violated + + def _action_rename_field(self, frontmatter: dict, rule: dict) -> Tuple[dict, bool]: + """Rename a specific field""" + from_field = rule.get('from') + to_field = rule.get('to') + + if not from_field or not to_field: + return frontmatter, False + + modified = dict(frontmatter) + + if from_field in modified: + modified[to_field] = modified.pop(from_field) + return modified, True + + return modified, False + + def _action_normalize_value(self, value: Any, rule: dict) -> Tuple[Any, bool]: + """Normalize a value using regex pattern(s) or transform""" + pattern = rule.get('pattern', '') + replacement = rule.get('replacement', '') + transform = rule.get('transform', '') + split_on = rule.get('split_on', '') + + violated = False + + # Handle comma-separated string first (convert to list) + if split_on and isinstance(value, str): + items = [item.strip() for item in value.split(split_on)] + normalized = [] + for item in items: + if transform == 'lower': + normalized_item = item.lower() + elif transform == 'upper': + normalized_item = item.upper() + elif pattern: + normalized_item = re.sub( + pattern, + replacement, + item, + flags=re.MULTILINE if rule.get('multiline') else 0 + ) + else: + normalized_item = item + + if normalized_item: + normalized.append(normalized_item) + violated = True # Format changed from string to list + return normalized, violated + + # Handle list values (like tags) + elif isinstance(value, list): + normalized = [] + for item in value: + item_str = str(item) + + # Apply transform first + if transform == 'lower': + normalized_item = item_str.lower() + elif transform == 'upper': + normalized_item = item_str.upper() + elif pattern: + normalized_item = re.sub( + pattern, + replacement, + item_str, + flags=re.MULTILINE if rule.get('multiline') else 0 + ) + else: + normalized_item = item_str + + if normalized_item != item_str: + violated = True + if normalized_item: # Only keep non-empty + normalized.append(normalized_item) + return normalized, violated + + # Handle string values + elif isinstance(value, str): + if transform == 'lower': + normalized = value.lower() + elif transform == 'upper': + normalized = value.upper() + elif pattern: + normalized = re.sub( + pattern, + replacement, + value, + flags=re.MULTILINE if rule.get('multiline') else 0 + ) + else: + normalized = value + + violated = normalized != value + return normalized, violated + + return value, False + + def _action_validate(self, value: Any, rule: dict) -> bool: + """Validate a value against regex pattern""" + pattern = rule.get('pattern', '') + + if not pattern: + return True + + # Handle list values + if isinstance(value, list): + for item in value: + if not re.match(pattern, str(item), flags=re.MULTILINE if rule.get('multiline') else 0): + return False + return True + + # Handle string values + if isinstance(value, str): + return bool(re.match(pattern, value, flags=re.MULTILINE if rule.get('multiline') else 0)) + + return True + + def get_llm_prompt_for_field(self, field: str) -> str: + """Get LLM prompt text for a field if defined in rules""" + if field in self.rules: + for rule in self.rules[field]: + llm_prompt = rule.get('llm_prompt') + if llm_prompt: + return llm_prompt + + return "" diff --git a/core/template.py b/core/template.py index c3b0158..4fed6ea 100644 --- a/core/template.py +++ b/core/template.py @@ -1,9 +1,9 @@ """ -Template management +Template management with type support """ import json from pathlib import Path -from typing import Optional +from typing import Optional, Tuple class TemplateManager: @@ -11,7 +11,7 @@ class TemplateManager: self.templates_dir = templates_dir def load_template(self, name: str) -> dict: - """Load template by name""" + """Load template by name and parse field types""" template_path = self.templates_dir / f'{name}.json' if not template_path.exists(): @@ -19,25 +19,70 @@ class TemplateManager: template_path = self.templates_dir / 'default.json' with open(template_path, 'r', encoding='utf-8') as f: - return json.load(f) + raw_template = json.load(f) + + # Parse template to separate strategies and types + template = {} + for field, value in raw_template.items(): + template[field] = value + + return template + + def parse_field(self, field_value: str) -> Tuple[str, str]: + """ + Parse field value to extract strategy and type + Format: "strategy|strategy:type" or just "strategy|strategy" + + Returns: (strategy, field_type) + """ + # Check if type is specified with colon + if ':' in field_value: + parts = field_value.rsplit(':', 1) + strategy = parts[0] + field_type = parts[1].strip() + else: + strategy = field_value + field_type = 'text' # Default type + + return strategy, field_type def get_structured_output_schema(self, template: dict) -> dict: - """Generate JSON schema for structured output""" + """Generate JSON schema for structured output based on field types""" properties = {} required = [] - for field in template.keys(): - if field == 'tags': + for field, field_value in template.items(): + strategy, field_type = self.parse_field(str(field_value)) + + # Map field type to JSON schema type + if field_type == 'list': properties[field] = { "type": "array", "items": {"type": "string"}, - "description": f"Field: {field}" + "description": f"Field: {field} (list of text)" } - else: + elif field_type == 'number': + properties[field] = { + "type": "number", + "description": f"Field: {field} (number)" + } + elif field_type == 'checkbox': + properties[field] = { + "type": "boolean", + "description": f"Field: {field} (true/false)" + } + elif field_type in ('date', 'datetime'): properties[field] = { "type": "string", - "description": f"Field: {field}" + "format": "date-time" if field_type == 'datetime' else "date", + "description": f"Field: {field} ({field_type})" } + else: # text or unspecified + properties[field] = { + "type": "string", + "description": f"Field: {field} (text)" + } + required.append(field) return { diff --git a/guide.md b/guide.md index df1ef7f..6b2b2c3 100644 --- a/guide.md +++ b/guide.md @@ -1,11 +1,11 @@ --- title: Neural Networks Guide created: '2025-10-23T18:06:22+02:00' -changed: '2025-10-23T18:06:22+02:00' -authors: -- Test User -version: 44ed278 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Neural Networks Guide diff --git a/madomeda_changelog.txt b/madomeda_changelog.txt index e18d06e..9a3b045 100644 --- a/madomeda_changelog.txt +++ b/madomeda_changelog.txt @@ -49,3 +49,45 @@ 2025-10-23 19:23:44 (commit: ab4b95c664427f6b08bd7510fd8c925403887ea5) ai_test.md - frontmatter updated +2025-10-23 19:36:41 (commit: 245530389de0de900f95411aa899306e48051bef) + ADD_ONLY_EXAMPLES.md - frontmatter updated + CONFIGURATION.md - frontmatter updated + GETTING_STARTED.md - frontmatter updated + IMPLEMENTATION.md - frontmatter updated + PROJECT_SUMMARY.md - frontmatter updated + README.md - frontmatter updated + STRUCTURE.md - frontmatter updated + USAGE.md - frontmatter updated + VERSION_FIELD.md - frontmatter updated + advanced.md - frontmatter updated + ai_test.md - frontmatter updated + comparison.md - frontmatter updated + guide.md - frontmatter updated + sample.md - frontmatter updated + test.md - frontmatter updated + +2025-10-23 19:49:57 (commit: 3cde344ac405dbbcfbf9e19faaef9177155de1b8) + rules_test.md - frontmatter updated + +2025-10-23 19:51:58 (commit: 3cde344ac405dbbcfbf9e19faaef9177155de1b8) + rules_test.md - frontmatter updated + +2025-10-23 20:11:52 (commit: 7cfe9499ee208f03c0feed9dce586a7335960421) + ADD_ONLY_EXAMPLES.md - frontmatter updated + CONFIGURATION.md - frontmatter updated + GETTING_STARTED.md - frontmatter updated + IMPLEMENTATION.md - frontmatter updated + PROJECT_SUMMARY.md - frontmatter updated + README.md - frontmatter updated + STRUCTURE.md - frontmatter updated + USAGE.md - frontmatter updated + VERSION_FIELD.md - frontmatter updated + advanced.md - frontmatter updated + ai_test.md - frontmatter updated + comparison.md - frontmatter updated + guide.md - frontmatter updated + rules_test.md - frontmatter updated + sample.md - frontmatter updated + test.md - frontmatter updated + type_test.md - frontmatter updated + diff --git a/rules_test.md b/rules_test.md index afda65d..b8a5008 100644 --- a/rules_test.md +++ b/rules_test.md @@ -1,9 +1,16 @@ --- -Title: Rules Test -Tag: Machine-Learning, Deep Learning, Neural Networks, AI/ML -Summary: Testing the new rule-based normalization +title: Rules Test Document +created: '2025-10-23T19:49:52+02:00' +status: draft +priority: 5 +published: false +tags: +- machine_learning +- deep_learning +- neural_networks +- aiml +reviewers: [] --- - # Rules Test Document This tests the new self-contained rules system. diff --git a/sample.md b/sample.md index 33938f0..6ca84e9 100644 --- a/sample.md +++ b/sample.md @@ -1,11 +1,11 @@ --- title: Sample Document created: '2025-10-23T17:27:16+02:00' -changed: '2025-10-23T17:27:16+02:00' -authors: -- Test User -version: 34d2d05 +status: draft +priority: 5 +published: false tags: [] +reviewers: [] --- # Sample Document diff --git a/test.md b/test.md index ab86ef8..09b4c83 100644 --- a/test.md +++ b/test.md @@ -1,14 +1,14 @@ --- title: Test Document created: '2025-10-23T17:27:16+02:00' -changed: '2025-10-23T17:27:16+02:00' -authors: -- Test User -version: 34d2d05 +status: draft +priority: 5 +published: false tags: - python - machine_learning - data_science +reviewers: [] --- # Test Document diff --git a/type_test.md b/type_test.md index 7aaabb8..59fe52b 100644 --- a/type_test.md +++ b/type_test.md @@ -1,3 +1,11 @@ +--- +title: Type Test Document +created: '2025-10-23T20:11:45+02:00' +status: draft +priority: 5 +published: false +reviewers: [] +--- # Type Test Document Testing template field types.