DSPy로 문서에서 코드를 자동 생성하기
DSPy로 문서에서 코드를 자동 생성하기
이 튜토리얼에서는 DSPy를 사용해 어떤 라이브러리의 문서든 URL에서 자동으로 가져와 실제 동작하는 코드 예제를 생성하는 방법을 보여드릴게요. 이 시스템은 문서 웹사이트를 분석하고 핵심 개념을 추출한 뒤, 사용 사례에 맞는 코드 예제를 만들어 줘요.
출처: 문서
본문
무엇을 만들까요
문서 기반 코드 생성 시스템으로, 이런 일을 해요:
- 여러 URL에서 문서를 가져와 파싱
- API 패턴, 메서드, 사용 예제 추출
- 특정 사용 사례에 맞는 동작 코드 생성
- 설명과 모범 사례 제공
- 어떤 라이브러리의 문서와도 동작
설정 (Setup)
pip install dspy requests beautifulsoup4 html2text
1단계: 문서 가져오기와 처리
문서를 가져오는 DocumentationFetcher, 문서를 분석하는 시그니처들, 그리고 코드를 생성하는 시그니처들로 시스템을 구성해요.
import dspy
import requests
from bs4 import BeautifulSoup
import html2text
from typing import List, Dict, Any
import json
from urllib.parse import urljoin, urlparse
import time
# Configure DSPy
lm = dspy.LM(model='openai/gpt-4o-mini')
dspy.configure(lm=lm)
class DocumentationFetcher:
"""Fetches and processes documentation from URLs."""
def __init__(self, max_retries=3, delay=1):
self.session = requests.Session()
self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
})
self.max_retries = max_retries
self.delay = delay
self.html_converter = html2text.HTML2Text()
self.html_converter.ignore_links = False
self.html_converter.ignore_images = True
def fetch_url(self, url: str) -> dict[str, str]:
"""Fetch content from a single URL."""
for attempt in range(self.max_retries):
try:
print(f"📡 Fetching: {url} (attempt {attempt + 1})")
response = self.session.get(url, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser')
# Remove script and style elements
for script in soup(["script", "style", "nav", "footer", "header"]):
script.decompose()
# Convert to markdown for better LLM processing
markdown_content = self.html_converter.handle(str(soup))
return {
"url": url,
"title": soup.title.string if soup.title else "No title",
"content": markdown_content,
"success": True
}
except Exception as e:
print(f"❌ Error fetching {url}: {e}")
if attempt < self.max_retries - 1:
time.sleep(self.delay)
else:
return {
"url": url,
"title": "Failed to fetch",
"content": f"Error: {str(e)}",
"success": False
}
return {"url": url, "title": "Failed", "content": "", "success": False}
def fetch_documentation(self, urls: list[str]) -> list[dict[str, str]]:
"""Fetch documentation from multiple URLs."""
results = []
for url in urls:
result = self.fetch_url(url)
results.append(result)
time.sleep(self.delay) # Be respectful to servers
return results
class LibraryAnalyzer(dspy.Signature):
"""Analyze library documentation to understand core concepts and patterns."""
library_name: str = dspy.InputField(desc="Name of the library to analyze")
documentation_content: str = dspy.InputField(desc="Combined documentation content")
core_concepts: list[str] = dspy.OutputField(desc="Main concepts and components")
common_patterns: list[str] = dspy.OutputField(desc="Common usage patterns")
key_methods: list[str] = dspy.OutputField(desc="Important methods and functions")
installation_info: str = dspy.OutputField(desc="Installation and setup information")
code_examples: list[str] = dspy.OutputField(desc="Example code snippets found")
class CodeGenerator(dspy.Signature):
"""Generate code examples for specific use cases using the target library."""
library_info: str = dspy.InputField(desc="Library concepts and patterns")
use_case: str = dspy.InputField(desc="Specific use case to implement")
requirements: str = dspy.InputField(desc="Additional requirements or constraints")
code_example: str = dspy.OutputField(desc="Complete, working code example")
explanation: str = dspy.OutputField(desc="Step-by-step explanation of the code")
best_practices: list[str] = dspy.OutputField(desc="Best practices and tips")
imports_needed: list[str] = dspy.OutputField(desc="Required imports and dependencies")
class DocumentationLearningAgent(dspy.Module):
"""Agent that learns from documentation URLs and generates code examples."""
def __init__(self):
super().__init__()
self.fetcher = DocumentationFetcher()
self.analyze_docs = dspy.ChainOfThought(LibraryAnalyzer)
self.generate_code = dspy.ChainOfThought(CodeGenerator)
self.refine_code = dspy.ChainOfThought(
"code, feedback -> improved_code: str, changes_made: list[str]"
)
def learn_from_urls(self, library_name: str, doc_urls: list[str]) -> Dict:
"""Learn about a library from its documentation URLs."""
print(f"📚 Learning about {library_name} from {len(doc_urls)} URLs...")
# Fetch all documentation
docs = self.fetcher.fetch_documentation(doc_urls)
# Combine successful fetches
combined_content = "\n\n---\n\n".join([
f"URL: {doc['url']}\nTitle: {doc['title']}\n\n{doc['content']}"
for doc in docs if doc['success']
])
if not combined_content:
raise ValueError("No documentation could be fetched successfully")
# Analyze combined documentation
analysis = self.analyze_docs(
library_name=library_name,
documentation_content=combined_content
)
return {
"library": library_name,
"source_urls": [doc['url'] for doc in docs if doc['success']],
"core_concepts": analysis.core_concepts,
"patterns": analysis.common_patterns,
"methods": analysis.key_methods,
"installation": analysis.installation_info,
"examples": analysis.code_examples,
"fetched_docs": docs
}
def generate_example(self, library_info: Dict, use_case: str, requirements: str = "") -> Dict:
"""Generate a code example for a specific use case."""
# Format library information for the generator
info_text = f"""
Library: {library_info['library']}
Core Concepts: {', '.join(library_info['core_concepts'])}
Common Patterns: {', '.join(library_info['patterns'])}
Key Methods: {', '.join(library_info['methods'])}
Installation: {library_info['installation']}
Example Code Snippets: {'; '.join(library_info['examples'][:3])} # First 3 examples
"""
code_result = self.generate_code(
library_info=info_text,
use_case=use_case,
requirements=requirements
)
return {
"code": code_result.code_example,
"explanation": code_result.explanation,
"best_practices": code_result.best_practices,
"imports": code_result.imports_needed
}
# Initialize the learning agent
agent = DocumentationLearningAgent()
DocumentationFetcher는 HTML을 마크다운으로 변환해서 LLM이 다루기 좋게 만들어 줘요. LibraryAnalyzer는 문서에서 핵심 개념을 뽑아내고, CodeGenerator는 그 정보를 바탕으로 실제 코드를 만들어 내요.
2단계: 문서 URL에서 학습하기
learn_library_from_urls 함수로 라이브러리 문서 URL들을 넣어 학습시켜 볼게요.
def learn_library_from_urls(library_name: str, documentation_urls: list[str]) -> Dict:
"""Learn about any library from its documentation URLs."""
try:
library_info = agent.learn_from_urls(library_name, documentation_urls)
print(f"\n🔍 Library Analysis Results for {library_name}:")
print(f"Sources: {len(library_info['source_urls'])} successful fetches")
print(f"Core Concepts: {library_info['core_concepts']}")
print(f"Common Patterns: {library_info['patterns']}")
print(f"Key Methods: {library_info['methods']}")
print(f"Installation: {library_info['installation']}")
print(f"Found {len(library_info['examples'])} code examples")
return library_info
except Exception as e:
print(f"❌ Error learning library: {e}")
raise
# Example 1: Learn FastAPI from official documentation
fastapi_urls = [
"https://fastapi.tiangolo.com/",
"https://fastapi.tiangolo.com/tutorial/first-steps/",
"https://fastapi.tiangolo.com/tutorial/path-params/",
"https://fastapi.tiangolo.com/tutorial/query-params/"
]
print("🚀 Learning FastAPI from official documentation...")
fastapi_info = learn_library_from_urls("FastAPI", fastapi_urls)
# Example 2: Learn a different library (you can replace with any library)
streamlit_urls = [
"https://docs.streamlit.io/",
"https://docs.streamlit.io/get-started",
"https://docs.streamlit.io/develop/api-reference"
]
print("\n\n📊 Learning Streamlit from official documentation...")
streamlit_info = learn_library_from_urls("Streamlit", streamlit_urls)
이 코드는 FastAPI와 Streamlit 두 라이브러리의 공식 문서를 가져와 학습해요. URL만 바꾸면 어떤 라이브러리든 적용할 수 있어요.
3단계: 코드 예제 생성하기
학습한 정보를 바탕으로 사용 사례별 코드 예제를 만들어 볼게요. 기본 설정, 일반 연산, 고급 사용법 세 가지 사용 사례를 정의했어요.
def generate_examples_for_library(library_info: Dict, library_name: str):
"""Generate code examples for any library based on its documentation."""
# Define generic use cases that can apply to most libraries
use_cases = [
{
"name": "Basic Setup and Hello World",
"description": f"Create a minimal working example with {library_name}",
"requirements": "Include installation, imports, and basic usage"
},
{
"name": "Common Operations",
"description": f"Demonstrate the most common {library_name} operations",
"requirements": "Show typical workflow and best practices"
},
{
"name": "Advanced Usage",
"description": f"Create a more complex example showcasing {library_name} capabilities",
"requirements": "Include error handling and optimization"
}
]
generated_examples = []
print(f"\n🔧 Generating examples for {library_name}...")
for use_case in use_cases:
print(f"\n📝 {use_case['name']}")
print(f"Description: {use_case['description']}")
example = agent.generate_example(
library_info=library_info,
use_case=use_case['description'],
requirements=use_case['requirements']
)
print("\n💻 Generated Code:")
print("```python")
print(example['code'])
print("```")
print("\n📦 Required Imports:")
for imp in example['imports']:
print(f" • {imp}")
print("\n📝 Explanation:")
print(example['explanation'])
print("\n✅ Best Practices:")
for practice in example['best_practices']:
print(f" • {practice}")
generated_examples.append({
"use_case": use_case['name'],
"code": example['code'],
"imports": example['imports'],
"explanation": example['explanation'],
"best_practices": example['best_practices']
})
print("-" * 80)
return generated_examples
# Generate examples for both libraries
print("🎯 Generating FastAPI Examples:")
fastapi_examples = generate_examples_for_library(fastapi_info, "FastAPI")
print("\n\n🎯 Generating Streamlit Examples:")
streamlit_examples = generate_examples_for_library(streamlit_info, "Streamlit")
4단계: 대화형 라이브러리 학습 함수
터미널에서 사용자 입력을 받아 어떤 라이브러리든 학습하고 예제를 만들어 주는 대화형 세션도 만들 수 있어요.
def learn_any_library(library_name: str, documentation_urls: list[str], use_cases: list[str] = None):
"""Learn any library from its documentation and generate examples."""
if use_cases is None:
use_cases = [
"Basic setup and hello world example",
"Common operations and workflows",
"Advanced usage with best practices"
]
print(f"🚀 Starting automated learning for {library_name}...")
print(f"Documentation sources: {len(documentation_urls)} URLs")
try:
# Step 1: Learn from documentation
library_info = agent.learn_from_urls(library_name, documentation_urls)
# Step 2: Generate examples for each use case
all_examples = []
for i, use_case in enumerate(use_cases, 1):
print(f"\n📝 Generating example {i}/{len(use_cases)}: {use_case}")
example = agent.generate_example(
library_info=library_info,
use_case=use_case,
requirements="Include error handling, comments, and follow best practices"
)
all_examples.append({
"use_case": use_case,
"code": example['code'],
"imports": example['imports'],
"explanation": example['explanation'],
"best_practices": example['best_practices']
})
return {
"library_info": library_info,
"examples": all_examples
}
except Exception as e:
print(f"❌ Error learning {library_name}: {e}")
return None
def interactive_learning_session():
"""Interactive session for learning libraries with user input."""
print("🎯 Welcome to the Interactive Library Learning System!")
print("This system will help you learn any Python library from its documentation.\n")
learned_libraries = {}
while True:
print("\n" + "="*60)
print("🚀 LIBRARY LEARNING SESSION")
print("="*60)
# Get library name from user
library_name = input("\n📚 Enter the library name you want to learn (or 'quit' to exit): ").strip()
if library_name.lower() in ['quit', 'exit', 'q']:
print("\n👋 Thanks for using the Interactive Library Learning System!")
break
if not library_name:
print("❌ Please enter a valid library name.")
continue
# Get documentation URLs
print(f"\n🔗 Enter documentation URLs for {library_name} (one per line, empty line to finish):")
urls = []
while True:
url = input(" URL: ").strip()
if not url:
break
if not url.startswith(('http://', 'https://')):
print(" ⚠️ Please enter a valid URL starting with http:// or https://")
continue
urls.append(url)
if not urls:
print("❌ No valid URLs provided. Skipping this library.")
continue
# Get custom use cases from user
print(f"\n🎯 Define use cases for {library_name} (optional, press Enter for defaults):")
print(" Default use cases will be: Basic setup, Common operations, Advanced usage")
user_wants_custom = input(" Do you want to define custom use cases? (y/n): ").strip().lower()
use_cases = None
if user_wants_custom in ['y', 'yes']:
print(" Enter your use cases (one per line, empty line to finish):")
use_cases = []
while True:
use_case = input(" Use case: ").strip()
if not use_case:
break
use_cases.append(use_case)
if not use_cases:
print(" No custom use cases provided, using defaults.")
use_cases = None
# Learn the library
print(f"\n🚀 Starting learning process for {library_name}...")
result = learn_any_library(library_name, urls, use_cases)
if result:
learned_libraries[library_name] = result
print(f"\n✅ Successfully learned {library_name}!")
# Show summary
print(f"\n📊 Learning Summary for {library_name}:")
print(f" • Core concepts: {len(result['library_info']['core_concepts'])} identified")
print(f" • Common patterns: {len(result['library_info']['patterns'])} found")
print(f" • Examples generated: {len(result['examples'])}")
# Ask if user wants to see examples
show_examples = input(f"\n👀 Do you want to see the generated examples for {library_name}? (y/n): ").strip().lower()
if show_examples in ['y', 'yes']:
for i, example in enumerate(result['examples'], 1):
print(f"\n{'─'*50}")
print(f"📝 Example {i}: {example['use_case']}")
print(f"{'─'*50}")
print("\n💻 Generated Code:")
print("```python")
print(example['code'])
print("```")
print(f"\n📦 Required Imports:")
for imp in example['imports']:
print(f" • {imp}")
print(f"\n📝 Explanation:")
print(example['explanation'])
print(f"\n✅ Best Practices:")
for practice in example['best_practices']:
print(f" • {practice}")
# Ask if user wants to see the next example
if i < len(result['examples']):
continue_viewing = input(f"\nContinue to next example? (y/n): ").strip().lower()
if continue_viewing not in ['y', 'yes']:
break
# Offer to save results
save_results = input(f"\n💾 Save learning results for {library_name} to file? (y/n): ").strip().lower()
if save_results in ['y', 'yes']:
filename = input(f" Enter filename (default: {library_name.lower()}_learning.json): ").strip()
if not filename:
filename = f"{library_name.lower()}_learning.json"
try:
import json
with open(filename, 'w') as f:
json.dump(result, f, indent=2, default=str)
print(f" ✅ Results saved to {filename}")
except Exception as e:
print(f" ❌ Error saving file: {e}")
else:
print(f"❌ Failed to learn {library_name}")
# Ask if user wants to learn another library
print(f"\n📚 Libraries learned so far: {list(learned_libraries.keys())}")
continue_learning = input("\n🔄 Do you want to learn another library? (y/n): ").strip().lower()
if continue_learning not in ['y', 'yes']:
break
# Final summary
if learned_libraries:
print(f"\n🎉 Session Summary:")
print(f"Successfully learned {len(learned_libraries)} libraries:")
for lib_name, info in learned_libraries.items():
print(f" • {lib_name}: {len(info['examples'])} examples generated")
return learned_libraries
# Example: Run interactive learning session
if __name__ == "__main__":
# Run interactive session
learned_libraries = interactive_learning_session()
예시 출력 (Example Output)
대화형 학습 시스템을 실행하면 이런 과정이 펼쳐져요:
대화형 세션 시작:
🎯 Welcome to the Interactive Library Learning System!
This system will help you learn any Python library from its documentation.
============================================================
🚀 LIBRARY LEARNING SESSION
============================================================
📚 Enter the library name you want to learn (or 'quit' to exit): FastAPI
🔗 Enter documentation URLs for FastAPI (one per line, empty line to finish):
URL: https://fastapi.tiangolo.com/
URL: https://fastapi.tiangolo.com/tutorial/first-steps/
URL: https://fastapi.tiangolo.com/tutorial/path-params/
URL:
🎯 Define use cases for FastAPI (optional, press Enter for defaults):
Default use cases will be: Basic setup, Common operations, Advanced usage
Do you want to define custom use cases? (y/n): y
Enter your use cases (one per line, empty line to finish):
Use case: Create a REST API with authentication
Use case: Build a file upload endpoint
Use case: Add database integration with SQLAlchemy
Use case:
문서 처리:
🚀 Starting learning process for FastAPI...
🚀 Starting automated learning for FastAPI...
Documentation sources: 3 URLs
📡 Fetching: https://fastapi.tiangolo.com/ (attempt 1)
📡 Fetching: https://fastapi.tiangolo.com/tutorial/first-steps/ (attempt 1)
📡 Fetching: https://fastapi.tiangolo.com/tutorial/path-params/ (attempt 1)
📚 Learning about FastAPI from 3 URLs...
🔍 Library Analysis Results for FastAPI:
Sources: 3 successful fetches
Core Concepts: ['FastAPI app', 'path operations', 'dependencies', 'request/response models']
Common Patterns: ['app = FastAPI()', 'decorator-based routing', 'Pydantic models']
Key Methods: ['FastAPI()', '@app.get()', '@app.post()', 'uvicorn.run()']
Installation: pip install fastapi uvicorn
코드 생성:
📝 Generating example 1/3: Create a REST API with authentication
✅ Successfully learned FastAPI!
📊 Learning Summary for FastAPI:
• Core concepts: 4 identified
• Common patterns: 3 found
• Examples generated: 3
👀 Do you want to see the generated examples for FastAPI? (y/n): y
──────────────────────────────────────────────────
📝 Example 1: Create a REST API with authentication
──────────────────────────────────────────────────
💻 Generated Code:
from fastapi import FastAPI, Depends, HTTPException, status
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
import uvicorn
from typing import Dict
import jwt
from datetime import datetime, timedelta
app = FastAPI(title="Authenticated API", version="1.0.0")
security = HTTPBearer()
# Secret key for JWT (use environment variable in production)
SECRET_KEY = "your-secret-key-here"
ALGORITHM = "HS256"
def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
try:
payload = jwt.decode(credentials.credentials, SECRET_KEY, algorithms=[ALGORITHM])
username: str = payload.get("sub")
if username is None:
raise HTTPException(status_code=401, detail="Invalid token")
return username
except jwt.PyJWTError:
raise HTTPException(status_code=401, detail="Invalid token")
@app.post("/login")
async def login(username: str, password: str) -> dict[str, str]:
# In production, verify against database
if username == "admin" and password == "secret":
token_data = {"sub": username, "exp": datetime.utcnow() + timedelta(hours=24)}
token = jwt.encode(token_data, SECRET_KEY, algorithm=ALGORITHM)
return {"access_token": token, "token_type": "bearer"}
raise HTTPException(status_code=401, detail="Invalid credentials")
@app.get("/protected")
async def protected_route(current_user: str = Depends(verify_token)) -> dict[str, str]:
return {"message": f"Hello {current_user}! This is a protected route."}
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)
📦 Required Imports:
• pip install fastapi uvicorn python-jose[cryptography]
• from fastapi import FastAPI, Depends, HTTPException, status
• from fastapi.security import HTTPBearer
• import jwt
📝 Explanation:
This example creates a FastAPI application with JWT-based authentication. It includes a login endpoint that returns a JWT token and a protected route that requires authentication...
✅ Best Practices:
• Use environment variables for secret keys
• Implement proper password hashing in production
• Add token expiration and refresh logic
• Include proper error handling
Continue to next example? (y/n): n
💾 Save learning results for FastAPI to file? (y/n): y
Enter filename (default: fastapi_learning.json):
✅ Results saved to fastapi_learning.json
📚 Libraries learned so far: ['FastAPI']
🔄 Do you want to learn another library? (y/n): n
🎉 Session Summary:
Successfully learned 1 libraries:
• FastAPI: 3 examples generated
다음 단계
- GitHub 통합: README 파일과 예제 저장소에서 학습
- 비디오 튜토리얼 처리: 영상 문서에서 정보 추출
- 커뮤니티 예제: Stack Overflow와 포럼에서 예제 집계
- 버전 비교: 라이브러리 버전 간 API 변경 추적
- 테스트 생성: 생성된 코드에 대한 단위 테스트 자동 생성
- 페이지 크롤링: 문서 페이지를 자동 크롤링해 사용법을 적극적으로 파악
이 튜토리얼은 DSPy가 낯선 라이브러리를 문서로부터 학습하는 전체 과정을 자동화할 수 있음을 보여줘요. 빠른 기술 도입과 탐색에 아주 유용하죠.