feat: Add Weaviate memory export scripts and documentation
Added two export scripts to backup memory collections: 1. export_conversations.py: - Exports all Conversation + Message objects to markdown - Includes conversation metadata (category, timestamps, participants) - Formats messages chronologically with role indicators - Generated: docs/conversations.md (12 conversations, 377 messages) 2. export_thoughts.py: - Exports all Thought objects to markdown - Groups by thought_type with summary statistics - Includes metadata (trigger, emotional_state, concepts, privacy) - Generated: docs/thoughts.md (104 thoughts across 8 types) Both scripts use UTF-8 encoding for markdown output with emoji formatting for better readability. Exports stored in docs/ for versioned backup of memory collections. Stats: - Conversations: 12 (5 testing, 7 general) - Messages: 377 total - Thoughts: 104 (28 reflection, 36 synthesis, 27 test) - Privacy: 100% private Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
43324
docs/conversations.md
Normal file
43324
docs/conversations.md
Normal file
File diff suppressed because it is too large
Load Diff
3027
docs/thoughts.md
Normal file
3027
docs/thoughts.md
Normal file
File diff suppressed because it is too large
Load Diff
148
export_conversations.py
Normal file
148
export_conversations.py
Normal file
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Export conversations from Weaviate to Markdown file.
|
||||
|
||||
Exports all conversations with their messages to docs/conversations.md
|
||||
"""
|
||||
|
||||
import weaviate
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def export_conversations_to_md(output_file: str = "docs/conversations.md"):
|
||||
"""Export all conversations to markdown file."""
|
||||
|
||||
# Connect to Weaviate
|
||||
client = weaviate.connect_to_local()
|
||||
|
||||
try:
|
||||
# Get collections
|
||||
conversation_collection = client.collections.get("Conversation")
|
||||
message_collection = client.collections.get("Message")
|
||||
|
||||
# Fetch all conversations (sorted by start date)
|
||||
conversations_response = conversation_collection.query.fetch_objects(
|
||||
limit=1000
|
||||
)
|
||||
|
||||
conversations = conversations_response.objects
|
||||
print(f"Found {len(conversations)} conversations")
|
||||
|
||||
# Sort by timestamp_start
|
||||
conversations = sorted(
|
||||
conversations,
|
||||
key=lambda c: c.properties.get("timestamp_start", datetime.min),
|
||||
reverse=True
|
||||
)
|
||||
|
||||
# Build markdown content
|
||||
lines = []
|
||||
lines.append("# Conversations Export")
|
||||
lines.append(f"\n**Exported**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
lines.append(f"\n**Total conversations**: {len(conversations)}")
|
||||
lines.append("\n---\n")
|
||||
|
||||
# Process each conversation
|
||||
for idx, conv in enumerate(conversations, 1):
|
||||
props = conv.properties
|
||||
|
||||
conv_id = props.get("conversation_id", "unknown")
|
||||
category = props.get("category", "N/A")
|
||||
summary = props.get("summary", "No summary")
|
||||
timestamp_start = props.get("timestamp_start")
|
||||
timestamp_end = props.get("timestamp_end")
|
||||
participants = props.get("participants", [])
|
||||
tags = props.get("tags", [])
|
||||
message_count = props.get("message_count", 0)
|
||||
context = props.get("context", "")
|
||||
|
||||
# Format timestamps
|
||||
start_str = timestamp_start.strftime('%Y-%m-%d %H:%M:%S') if timestamp_start else "N/A"
|
||||
end_str = timestamp_end.strftime('%Y-%m-%d %H:%M:%S') if timestamp_end else "Ongoing"
|
||||
|
||||
# Write conversation header
|
||||
lines.append(f"## Conversation {idx}: {conv_id}")
|
||||
lines.append(f"\n**Category**: {category}")
|
||||
lines.append(f"**Start**: {start_str}")
|
||||
lines.append(f"**End**: {end_str}")
|
||||
|
||||
if participants:
|
||||
lines.append(f"**Participants**: {', '.join(participants)}")
|
||||
|
||||
if tags:
|
||||
lines.append(f"**Tags**: {', '.join(tags)}")
|
||||
|
||||
lines.append(f"**Message count**: {message_count}")
|
||||
|
||||
lines.append(f"\n**Summary**:\n{summary}")
|
||||
|
||||
if context:
|
||||
lines.append(f"\n**Context**:\n{context}")
|
||||
|
||||
# Fetch messages for this conversation
|
||||
messages_response = message_collection.query.fetch_objects(
|
||||
filters=weaviate.classes.query.Filter.by_property("conversation_id").equal(conv_id),
|
||||
limit=1000
|
||||
)
|
||||
|
||||
messages = messages_response.objects
|
||||
|
||||
# Sort by order_index
|
||||
messages = sorted(
|
||||
messages,
|
||||
key=lambda m: m.properties.get("order_index", 0)
|
||||
)
|
||||
|
||||
if messages:
|
||||
lines.append(f"\n### Messages ({len(messages)})\n")
|
||||
|
||||
for msg in messages:
|
||||
msg_props = msg.properties
|
||||
role = msg_props.get("role", "unknown")
|
||||
content = msg_props.get("content", "")
|
||||
timestamp = msg_props.get("timestamp")
|
||||
order_idx = msg_props.get("order_index", 0)
|
||||
|
||||
timestamp_str = timestamp.strftime('%H:%M:%S') if timestamp else "N/A"
|
||||
|
||||
# Format role emoji
|
||||
role_emoji = {
|
||||
"user": "👤",
|
||||
"assistant": "🤖",
|
||||
"system": "⚙️"
|
||||
}.get(role, "❓")
|
||||
|
||||
lines.append(f"**[{order_idx}] {role_emoji} {role.upper()}** ({timestamp_str})")
|
||||
lines.append(f"\n{content}\n")
|
||||
else:
|
||||
lines.append("\n*No messages found*\n")
|
||||
|
||||
lines.append("\n---\n")
|
||||
|
||||
# Write to file
|
||||
output_path = Path(output_file)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
print(f"\n[OK] Exported {len(conversations)} conversations to {output_file}")
|
||||
|
||||
# Stats
|
||||
total_messages = sum(c.properties.get("message_count", 0) for c in conversations)
|
||||
print(f" Total messages: {total_messages}")
|
||||
|
||||
categories = {}
|
||||
for c in conversations:
|
||||
cat = c.properties.get("category", "unknown")
|
||||
categories[cat] = categories.get(cat, 0) + 1
|
||||
|
||||
print(f" Categories: {dict(categories)}")
|
||||
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
export_conversations_to_md()
|
||||
140
export_thoughts.py
Normal file
140
export_thoughts.py
Normal file
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Export thoughts from Weaviate to Markdown file.
|
||||
|
||||
Exports all thoughts to docs/thoughts.md
|
||||
"""
|
||||
|
||||
import weaviate
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def export_thoughts_to_md(output_file: str = "docs/thoughts.md"):
|
||||
"""Export all thoughts to markdown file."""
|
||||
|
||||
# Connect to Weaviate
|
||||
client = weaviate.connect_to_local()
|
||||
|
||||
try:
|
||||
# Get collection
|
||||
thought_collection = client.collections.get("Thought")
|
||||
|
||||
# Fetch all thoughts (sorted by timestamp)
|
||||
thoughts_response = thought_collection.query.fetch_objects(
|
||||
limit=1000
|
||||
)
|
||||
|
||||
thoughts = thoughts_response.objects
|
||||
print(f"Found {len(thoughts)} thoughts")
|
||||
|
||||
# Sort by timestamp (most recent first)
|
||||
thoughts = sorted(
|
||||
thoughts,
|
||||
key=lambda t: t.properties.get("timestamp", datetime.min),
|
||||
reverse=True
|
||||
)
|
||||
|
||||
# Build markdown content
|
||||
lines = []
|
||||
lines.append("# Thoughts Export")
|
||||
lines.append(f"\n**Exported**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
lines.append(f"\n**Total thoughts**: {len(thoughts)}")
|
||||
lines.append("\n---\n")
|
||||
|
||||
# Group by type
|
||||
thoughts_by_type = {}
|
||||
for thought in thoughts:
|
||||
thought_type = thought.properties.get("thought_type", "unknown")
|
||||
if thought_type not in thoughts_by_type:
|
||||
thoughts_by_type[thought_type] = []
|
||||
thoughts_by_type[thought_type].append(thought)
|
||||
|
||||
# Write summary by type
|
||||
lines.append("## Summary by Type\n")
|
||||
for thought_type in sorted(thoughts_by_type.keys()):
|
||||
count = len(thoughts_by_type[thought_type])
|
||||
lines.append(f"- **{thought_type}**: {count}")
|
||||
|
||||
lines.append("\n---\n")
|
||||
|
||||
# Process each thought
|
||||
for idx, thought in enumerate(thoughts, 1):
|
||||
props = thought.properties
|
||||
|
||||
content = props.get("content", "No content")
|
||||
thought_type = props.get("thought_type", "unknown")
|
||||
timestamp = props.get("timestamp")
|
||||
trigger = props.get("trigger", "")
|
||||
emotional_state = props.get("emotional_state", "")
|
||||
concepts = props.get("concepts", [])
|
||||
privacy_level = props.get("privacy_level", "private")
|
||||
context = props.get("context", "")
|
||||
|
||||
# Format timestamp
|
||||
timestamp_str = timestamp.strftime('%Y-%m-%d %H:%M:%S') if timestamp else "N/A"
|
||||
|
||||
# Type emoji
|
||||
type_emoji = {
|
||||
"reflexion": "💭",
|
||||
"question": "❓",
|
||||
"intuition": "💡",
|
||||
"observation": "👁️",
|
||||
"conclusion": "✅",
|
||||
"hypothesis": "🤔",
|
||||
"discovery": "🔍"
|
||||
}.get(thought_type, "📝")
|
||||
|
||||
# Privacy emoji
|
||||
privacy_emoji = {
|
||||
"private": "🔒",
|
||||
"shared": "👥",
|
||||
"public": "🌐"
|
||||
}.get(privacy_level, "❓")
|
||||
|
||||
# Write thought entry
|
||||
lines.append(f"## {type_emoji} Thought {idx}: {thought_type.upper()}")
|
||||
lines.append(f"\n**Timestamp**: {timestamp_str}")
|
||||
lines.append(f"**Privacy**: {privacy_emoji} {privacy_level}")
|
||||
|
||||
if trigger:
|
||||
lines.append(f"**Trigger**: {trigger}")
|
||||
|
||||
if emotional_state:
|
||||
lines.append(f"**Emotional state**: {emotional_state}")
|
||||
|
||||
if concepts:
|
||||
lines.append(f"**Concepts**: {', '.join(concepts)}")
|
||||
|
||||
lines.append(f"\n### Content\n\n{content}\n")
|
||||
|
||||
if context:
|
||||
lines.append(f"**Context**: {context}\n")
|
||||
|
||||
lines.append("\n---\n")
|
||||
|
||||
# Write to file
|
||||
output_path = Path(output_file)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(lines))
|
||||
|
||||
print(f"\n[OK] Exported {len(thoughts)} thoughts to {output_file}")
|
||||
|
||||
# Stats
|
||||
print(f" Types: {dict((k, len(v)) for k, v in thoughts_by_type.items())}")
|
||||
|
||||
privacy_stats = {}
|
||||
for t in thoughts:
|
||||
privacy = t.properties.get("privacy_level", "unknown")
|
||||
privacy_stats[privacy] = privacy_stats.get(privacy, 0) + 1
|
||||
|
||||
print(f" Privacy: {dict(privacy_stats)}")
|
||||
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
export_thoughts_to_md()
|
||||
Reference in New Issue
Block a user