oval/madomeda
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improvements added to rules engine

Browse Source
This commit is contained in:
Test User
2025-10-23 20:14:21 +02:00
parent 7cfe9499ee
commit eea7ef0648
25 changed files with 1967 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/config.py
+90 -21
View File
@@ -76,12 +76,12 @@ OPENAI_API_KEY=""
"""Create default template"""
import json
default_template = {
"title": "heur|orig",
"created": "heur|orig",
"changed": "heur|orig",
"authors": "heur|orig",
"version": "heur|orig",
"tags": "heur|orig|ai default"
"title": "heur|orig:text",
"created": "heur|orig:datetime",
"changed": "heur|orig:datetime",
"authors": "heur|orig:list",
"version": "heur|orig:text",
"tags": "heur|orig|ai default:list"
}
with open(self.templates_dir / 'default.json', 'w', encoding='utf-8') as f:
json.dump(default_template, f, indent=2)
@@ -91,25 +91,94 @@ OPENAI_API_KEY=""
import json
rules = {
'tags_normalization.json': {
"name": "tags_normalization",
"description": "Normalize tags to lowercase with underscores",
"type": "tags"
},
'tag_to_tags.json': {
"name": "tag_to_tags",
"description": "Rename 'tag' key to 'tags'",
"type": "rename"
},
'lowercase_keys.json': {
'01_lowercase_keys.json': {
"name": "lowercase_keys",
"description": "All frontmatter keys must be lowercase",
"type": "keys"
"field": "*",
"priority": 1,
"action": "normalize_keys",
"transform": "lower",
"llm_prompt": "Ensure all keys are lowercase"
},
'summary_to_description.json': {
'02_tag_to_tags.json': {
"name": "tag_to_tags",
"description": "Rename 'tag' field to 'tags'",
"field": "tag",
"priority": 2,
"action": "rename_field",
"from": "tag",
"to": "tags",
"llm_prompt": "Convert tag field to tags array"
},
'03_summary_to_description.json': {
"name": "summary_to_description",
"description": "Rename 'summary' key to 'description'",
"type": "rename"
"description": "Rename 'summary' field to 'description'",
"field": "summary",
"priority": 2,
"action": "rename_field",
"from": "summary",
"to": "description",
"llm_prompt": "Convert summary to description"
},
'10_tags_format_list.json': {
"name": "tags_format_list",
"description": "Convert comma-separated tags to list",
"field": "tags",
"priority": 10,
"action": "normalize_value",
"split_on": ",",
"pattern": ".*",
"replacement": "\\g<0>",
"llm_prompt": "Ensure tags is a list, not a comma-separated string"
},
'20_tags_replace_spaces.json': {
"name": "tags_replace_spaces",
"description": "Replace spaces with underscores in tags",
"field": "tags",
"priority": 20,
"action": "normalize_value",
"pattern": " ",
"replacement": "_",
"llm_prompt": "Replace spaces with underscores in tags"
},
'21_tags_replace_hyphens.json': {
"name": "tags_replace_hyphens",
"description": "Replace hyphens with underscores in tags",
"field": "tags",
"priority": 21,
"action": "normalize_value",
"pattern": "-",
"replacement": "_",
"llm_prompt": "Replace hyphens with underscores in tags"
},
'30_tags_lowercase.json': {
"name": "tags_lowercase",
"description": "Convert tags to lowercase",
"field": "tags",
"priority": 30,
"action": "normalize_value",
"transform": "lower",
"llm_prompt": "Convert all tags to lowercase"
},
'40_tags_remove_invalid_chars.json': {
"name": "tags_remove_invalid_chars",
"description": "Remove characters that are not alphanumeric or underscore",
"field": "tags",
"priority": 40,
"action": "normalize_value",
"pattern": "[^a-z0-9_]",
"replacement": "",
"llm_prompt": "Remove any characters that are not lowercase letters, numbers, or underscores from tags"
},
'50_tags_validate_format.json': {
"name": "tags_validate_format",
"description": "Validate that tags only contain lowercase alphanumeric and underscores",
"field": "tags",
"priority": 50,
"action": "validate",
"pattern": "^[a-z0-9_]+$",
"multiline": False,
"llm_prompt": "Tags must contain only lowercase letters, numbers, and underscores"
}
}
core/processor.py
+37 -6
View File
@@ -7,7 +7,7 @@ from datetime import datetime
from core.database import Database
from core.repository import Repository
from core.frontmatter import FrontmatterParser
from core.rules import RulesEngine
from core.rules_processor import RulesProcessor
from core.template import TemplateManager
from core.llm import LLMClient
from core.changelog import ChangeLog
@@ -32,7 +32,7 @@ class FrontmatterProcessor:
api_config = self.config.get_api_config()
self.db = Database(repo.path / 'madomeda.db')
self.rules = RulesEngine(self.config.rules_dir)
self.rules = RulesProcessor(self.config.rules_dir)
self.templates = TemplateManager(self.config.templates_dir)
self.llm = LLMClient(self.config.prompts_dir, api_config)
self.changelog = ChangeLog(repo.path)
@@ -113,7 +113,7 @@ class FrontmatterProcessor:
)
# Apply rules
normalized_fm, is_conformant = self.rules.apply_rules(dict(original_fm))
normalized_fm, is_conformant, violations = self.rules.apply_rules(dict(original_fm))
# Check if we need to update
needs_update = self.force or not is_conformant or not self._matches_template(normalized_fm, template)
@@ -190,14 +190,21 @@ class FrontmatterProcessor:
new_fm = {}
all_tags = self.db.get_all_tags()
for field, strategy in template.items():
options = strategy.split('|')
for field, field_value in template.items():
# Parse field to get strategy and type
strategy_str, field_type = self.templates.parse_field(str(field_value))
options = strategy_str.split('|')
value = None
for option in options:
option = option.strip()
if option == 'heur':
if option.startswith('lit:'):
# Literal/static value
value = option[4:].strip()
# Convert based on type
value = self._convert_value_type(value, field_type)
elif option == 'heur':
# Use heuristics
value = self._get_heuristic_value(field, normalized_fm, body, file_path, git_info)
elif option == 'orig':
@@ -218,6 +225,30 @@ class FrontmatterProcessor:
return new_fm
def _convert_value_type(self, value: str, field_type: str):
"""Convert literal value to appropriate type"""
if field_type == 'checkbox':
return value.lower() in ('true', '1', 'yes', 'on')
elif field_type == 'number':
try:
if '.' in value:
return float(value)
return int(value)
except ValueError:
return value
elif field_type == 'list':
# Parse as JSON array or return empty list
if value == '[]':
return []
try:
import json
return json.loads(value)
except:
return [value]
else:
# text, date, datetime - keep as string
return value
def _get_heuristic_value(self, field: str, normalized_fm: dict, body: str,
file_path: Path, git_info: dict):
"""Get field value using heuristics"""
core/rules_processor.py
+255
View File
@@ -0,0 +1,255 @@
"""
Self-contained rules processor
Processes JSON-based rules without hardcoded logic
"""
import json
import re
from pathlib import Path
from typing import List, Tuple, Dict, Any
class RulesProcessor:
def __init__(self, rules_dir: Path):
self.rules_dir = rules_dir
self.rules = self._load_rules()
def _load_rules(self) -> Dict[str, List[dict]]:
"""Load all rule files and organize by field"""
rules_by_field = {}
if self.rules_dir.exists():
for rule_file in self.rules_dir.glob('*.json'):
with open(rule_file, 'r', encoding='utf-8') as f:
rule = json.load(f)
field = rule.get('field', '*')
if field not in rules_by_field:
rules_by_field[field] = []
rules_by_field[field].append(rule)
# Sort rules by priority (lower number = higher priority)
for field in rules_by_field:
rules_by_field[field].sort(key=lambda r: r.get('priority', 100))
return rules_by_field
def apply_rules(self, frontmatter: dict) -> Tuple[dict, bool, List[str]]:
"""
Apply all rules to frontmatter
Returns: (modified_fm, is_conformant, violations)
"""
if not frontmatter:
return frontmatter, True, []
modified = dict(frontmatter)
violations = []
# Process global rules (field: "*") first
if '*' in self.rules:
for rule in self.rules['*']:
modified, rule_violations = self._apply_rule(modified, rule, None)
violations.extend(rule_violations)
# Process rename rules to create new fields
for field in list(modified.keys()):
if field in self.rules:
for rule in self.rules[field]:
if rule.get('action') == 'rename_field':
modified, rule_violations = self._apply_rule(modified, rule, field)
violations.extend(rule_violations)
# Process other field-specific rules on current fields
for field in list(modified.keys()):
if field in self.rules:
for rule in self.rules[field]:
if rule.get('action') != 'rename_field': # Skip renames, already done
modified, rule_violations = self._apply_rule(modified, rule, field)
violations.extend(rule_violations)
is_conformant = len(violations) == 0
return modified, is_conformant, violations
def _apply_rule(self, frontmatter: dict, rule: dict, field: str = None) -> Tuple[dict, List[str]]:
"""Apply a single rule to frontmatter"""
modified = dict(frontmatter)
violations = []
action = rule.get('action', 'validate')
if action == 'normalize_keys':
modified, violated = self._action_normalize_keys(modified, rule)
if violated:
violations.append(rule.get('name', 'unknown'))
elif action == 'rename_field':
modified, violated = self._action_rename_field(modified, rule)
if violated:
violations.append(rule.get('name', 'unknown'))
elif action == 'normalize_value':
if field and field in modified:
modified[field], violated = self._action_normalize_value(modified[field], rule)
if violated:
violations.append(rule.get('name', 'unknown'))
elif action == 'validate':
if field and field in modified:
violated = not self._action_validate(modified[field], rule)
if violated:
violations.append(rule.get('name', 'unknown'))
return modified, violations
def _action_normalize_keys(self, frontmatter: dict, rule: dict) -> Tuple[dict, bool]:
"""Normalize all keys using regex pattern or transform"""
pattern = rule.get('pattern', '')
replacement = rule.get('replacement', '')
transform = rule.get('transform', '')
modified = {}
violated = False
for key, value in frontmatter.items():
new_key = key
# Apply transform (e.g., lower, upper)
if transform == 'lower':
new_key = key.lower()
elif transform == 'upper':
new_key = key.upper()
elif pattern:
# Apply regex pattern
new_key = re.sub(pattern, replacement, key, flags=re.MULTILINE if rule.get('multiline') else 0)
if new_key != key:
violated = True
modified[new_key] = value
return modified, violated
def _action_rename_field(self, frontmatter: dict, rule: dict) -> Tuple[dict, bool]:
"""Rename a specific field"""
from_field = rule.get('from')
to_field = rule.get('to')
if not from_field or not to_field:
return frontmatter, False
modified = dict(frontmatter)
if from_field in modified:
modified[to_field] = modified.pop(from_field)
return modified, True
return modified, False
def _action_normalize_value(self, value: Any, rule: dict) -> Tuple[Any, bool]:
"""Normalize a value using regex pattern(s) or transform"""
pattern = rule.get('pattern', '')
replacement = rule.get('replacement', '')
transform = rule.get('transform', '')
split_on = rule.get('split_on', '')
violated = False
# Handle comma-separated string first (convert to list)
if split_on and isinstance(value, str):
items = [item.strip() for item in value.split(split_on)]
normalized = []
for item in items:
if transform == 'lower':
normalized_item = item.lower()
elif transform == 'upper':
normalized_item = item.upper()
elif pattern:
normalized_item = re.sub(
pattern,
replacement,
item,
flags=re.MULTILINE if rule.get('multiline') else 0
)
else:
normalized_item = item
if normalized_item:
normalized.append(normalized_item)
violated = True # Format changed from string to list
return normalized, violated
# Handle list values (like tags)
elif isinstance(value, list):
normalized = []
for item in value:
item_str = str(item)
# Apply transform first
if transform == 'lower':
normalized_item = item_str.lower()
elif transform == 'upper':
normalized_item = item_str.upper()
elif pattern:
normalized_item = re.sub(
pattern,
replacement,
item_str,
flags=re.MULTILINE if rule.get('multiline') else 0
)
else:
normalized_item = item_str
if normalized_item != item_str:
violated = True
if normalized_item: # Only keep non-empty
normalized.append(normalized_item)
return normalized, violated
# Handle string values
elif isinstance(value, str):
if transform == 'lower':
normalized = value.lower()
elif transform == 'upper':
normalized = value.upper()
elif pattern:
normalized = re.sub(
pattern,
replacement,
value,
flags=re.MULTILINE if rule.get('multiline') else 0
)
else:
normalized = value
violated = normalized != value
return normalized, violated
return value, False
def _action_validate(self, value: Any, rule: dict) -> bool:
"""Validate a value against regex pattern"""
pattern = rule.get('pattern', '')
if not pattern:
return True
# Handle list values
if isinstance(value, list):
for item in value:
if not re.match(pattern, str(item), flags=re.MULTILINE if rule.get('multiline') else 0):
return False
return True
# Handle string values
if isinstance(value, str):
return bool(re.match(pattern, value, flags=re.MULTILINE if rule.get('multiline') else 0))
return True
def get_llm_prompt_for_field(self, field: str) -> str:
"""Get LLM prompt text for a field if defined in rules"""
if field in self.rules:
for rule in self.rules[field]:
llm_prompt = rule.get('llm_prompt')
if llm_prompt:
return llm_prompt
return ""
core/template.py
+55 -10
View File
@@ -1,9 +1,9 @@
"""
Template management
Template management with type support
"""
import json
from pathlib import Path
from typing import Optional
from typing import Optional, Tuple
class TemplateManager:
@@ -11,7 +11,7 @@ class TemplateManager:
self.templates_dir = templates_dir
def load_template(self, name: str) -> dict:
"""Load template by name"""
"""Load template by name and parse field types"""
template_path = self.templates_dir / f'{name}.json'
if not template_path.exists():
@@ -19,25 +19,70 @@ class TemplateManager:
template_path = self.templates_dir / 'default.json'
with open(template_path, 'r', encoding='utf-8') as f:
return json.load(f)
raw_template = json.load(f)
# Parse template to separate strategies and types
template = {}
for field, value in raw_template.items():
template[field] = value
return template
def parse_field(self, field_value: str) -> Tuple[str, str]:
"""
Parse field value to extract strategy and type
Format: "strategy|strategy:type" or just "strategy|strategy"
Returns: (strategy, field_type)
"""
# Check if type is specified with colon
if ':' in field_value:
parts = field_value.rsplit(':', 1)
strategy = parts[0]
field_type = parts[1].strip()
else:
strategy = field_value
field_type = 'text' # Default type
return strategy, field_type
def get_structured_output_schema(self, template: dict) -> dict:
"""Generate JSON schema for structured output"""
"""Generate JSON schema for structured output based on field types"""
properties = {}
required = []
for field in template.keys():
if field == 'tags':
for field, field_value in template.items():
strategy, field_type = self.parse_field(str(field_value))
# Map field type to JSON schema type
if field_type == 'list':
properties[field] = {
"type": "array",
"items": {"type": "string"},
"description": f"Field: {field}"
"description": f"Field: {field} (list of text)"
}
else:
elif field_type == 'number':
properties[field] = {
"type": "number",
"description": f"Field: {field} (number)"
}
elif field_type == 'checkbox':
properties[field] = {
"type": "boolean",
"description": f"Field: {field} (true/false)"
}
elif field_type in ('date', 'datetime'):
properties[field] = {
"type": "string",
"description": f"Field: {field}"
"format": "date-time" if field_type == 'datetime' else "date",
"description": f"Field: {field} ({field_type})"
}
else: # text or unspecified
properties[field] = {
"type": "string",
"description": f"Field: {field} (text)"
}
required.append(field)
return {