문서에서 코드 자동 생성하기
문서에서 코드 자동 생성하기 (Automated Code Generation from Documentation)
이 튜토리얼은 DSPy를 이용해 URL에서 문서를 자동으로 가져와 어떤 라이브러리든 동작하는 코드 예시를 만들어 내는 시스템을 소개해요. 문서 웹사이트를 분석하고 핵심 개념을 뽑아낸 뒤, 그 시나리오에 맞는 코드 예시를 생성해 줘요.
무엇을 만들게 되나요
문서 기반 코드 생성 시스템이 할 일은 이래요.
- 여러 URL에서 문서를 가져와 파싱
- API 패턴, 메서드, 사용 예시 추출
- 특정 사용 사례에 맞는 동작하는 코드 생성
- 설명과 모범 사례 제공
- 어떤 라이브러리의 문서와도 동작
설치
pip install dspy requests beautifulsoup4 html2text
Step 1: 문서 가져오기와 처리
import dspy
import requests
from bs4 import BeautifulSoup
import html2text
from typing import List, Dict, Any
import json
from urllib.parse import urljoin, urlparse
import time
# Configure DSPy
lm = dspy.LM(model='openai/gpt-4o-mini')
dspy.configure(lm=lm)
class DocumentationFetcher:
"""Fetches and processes documentation from URLs."""
def __init__(self, max_retries=3, delay=1):
self.session = requests.Session()
self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
})
self.max_retries = max_retries
self.delay = delay
self.html_converter = html2text.HTML2Text()
self.html_converter.ignore_links = False
self.html_converter.ignore_images = True
def fetch_url(self, url: str) -> dict[str, str]:
"""Fetch content from a single URL."""
for attempt in range(self.max_retries):
try:
print(f"📡 Fetching: {url} (attempt {attempt + 1})")
response = self.session.get(url, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser')
# Remove script and style elements
for script in soup(["script", "style", "nav", "footer", "header"]):
script.decompose()
# Convert to markdown for better LLM processing
markdown_content = self.html_converter.handle(str(soup))
return {
"url": url,
"title": soup.title.string if soup.title else "No title",
"content": markdown_content,
"success": True
}
except Exception as e:
print(f"❌ Error fetching {url}: {e}")
if attempt < self.max_retries - 1:
time.sleep(self.delay)
else:
return {
"url": url,
"title": "Failed to fetch",
"content": f"Error: {str(e)}",
"success": False
}
return {"url": url, "title": "Failed", "content": "", "success": False}
def fetch_documentation(self, urls: list[str]) -> list[dict[str, str]]:
"""Fetch documentation from multiple URLs."""
results = []
for url in urls:
result = self.fetch_url(url)
results.append(result)
time.sleep(self.delay) # Be respectful to servers
return results
class LibraryAnalyzer(dspy.Signature):
"""Analyze library documentation to understand core concepts and patterns."""
library_name: str = dspy.InputField(desc="Name of the library to analyze")
documentation_content: str = dspy.InputField(desc="Combined documentation content")
core_concepts: list[str] = dspy.OutputField(desc="Main concepts and components")
common_patterns: list[str] = dspy.OutputField(desc="Common usage patterns")
key_methods: list[str] = dspy.OutputField(desc="Important methods and functions")
installation_info: str = dspy.OutputField(desc="Installation and setup information")
code_examples: list[str] = dspy.OutputField(desc="Example code snippets found")
class CodeGenerator(dspy.Signature):
"""Generate code examples for specific use cases using the target library."""
library_info: str = dspy.InputField(desc="Library concepts and patterns")
use_case: str = dspy.InputField(desc="Specific use case to implement")
requirements: str = dspy.InputField(desc="Additional requirements or constraints")
code_example: str = dspy.OutputField(desc="Complete, working code example")
explanation: str = dspy.OutputField(desc="Step-by-step explanation of the code")
best_practices: list[str] = dspy.OutputField(desc="Best practices and tips")
imports_needed: list[str] = dspy.OutputField(desc="Required imports and dependencies")
class DocumentationLearningAgent(dspy.Module):
"""Agent that learns from documentation URLs and generates code examples."""
def __init__(self):
super().__init__()
self.fetcher = DocumentationFetcher()
self.analyze_docs = dspy.ChainOfThought(LibraryAnalyzer)
self.generate_code = dspy.ChainOfThought(CodeGenerator)
self.refine_code = dspy.ChainOfThought(
"code, feedback -> improved_code: str, changes_made: list[str]"
)
def learn_from_urls(self, library_name: str, doc_urls: list[str]) -> Dict:
"""Learn about a library from its documentation URLs."""
print(f"📚 Learning about {library_name} from {len(doc_urls)} URLs...")
# Fetch all documentation
docs = self.fetcher.fetch_documentation(doc_urls)
# Combine successful fetches
combined_content = "\n\n---\n\n".join([
f"URL: {doc['url']}\nTitle: {doc['title']}\n\n{doc['content']}"
for doc in docs if doc['success']
])
if not combined_content:
raise ValueError("No documentation could be fetched successfully")
# Analyze combined documentation
analysis = self.analyze_docs(
library_name=library_name,
documentation_content=combined_content
)
return {
"library": library_name,
"source_urls": [doc['url'] for doc in docs if doc['success']],
"core_concepts": analysis.core_concepts,
"patterns": analysis.common_patterns,
"methods": analysis.key_methods,
"installation": analysis.installation_info,
"examples": analysis.code_examples,
"fetched_docs": docs
}
def generate_example(self, library_info: Dict, use_case: str, requirements: str = "") -> Dict:
"""Generate a code example for a specific use case."""
# Format library information for the generator
info_text = f"""
Library: {library_info['library']}
Core Concepts: {', '.join(library_info['core_concepts'])}
Common Patterns: {', '.join(library_info['patterns'])}
Key Methods: {', '.join(library_info['methods'])}
Installation: {library_info['installation']}
Example Code Snippets: {'; '.join(library_info['examples'][:3])} # First 3 examples
"""
code_result = self.generate_code(
library_info=info_text,
use_case=use_case,
requirements=requirements
)
return {
"code": code_result.code_example,
"explanation": code_result.explanation,
"best_practices": code_result.best_practices,
"imports": code_result.imports_needed
}
# Initialize the learning agent
agent = DocumentationLearningAgent()
DocumentationFetcher는 요청을 재시도하고 HTML에서 nav·script 등을 제거한 뒤 markdown으로 변환해 LLM이 처리하기 좋게 만들어요. LibraryAnalyzer signature는 문서에서 핵심 개념·패턴·주요 메서드·설치 정보를 뽑고, CodeGenerator는 그 정보를 바탕으로 실제 코드 예시를 생성해요.
Step 2: 문서 URL에서 학습하기
def learn_library_from_urls(library_name: str, documentation_urls: list[str]) -> Dict:
"""Learn about any library from its documentation URLs."""
try:
library_info = agent.learn_from_urls(library_name, documentation_urls)
print(f"\n🔍 Library Analysis Results for {library_name}:")
print(f"Sources: {len(library_info['source_urls'])} successful fetches")
print(f"Core Concepts: {library_info['core_concepts']}")
print(f"Common Patterns: {library_info['patterns']}")
print(f"Key Methods: {library_info['methods']}")
print(f"Installation: {library_info['installation']}")
print(f"Found {len(library_info['examples'])} code examples")
return library_info
except Exception as e:
print(f"❌ Error learning library: {e}")
raise
# Example 1: Learn FastAPI from official documentation
fastapi_urls = [
"https://fastapi.tiangolo.com/",
"https://fastapi.tiangolo.com/tutorial/first-steps/",
"https://fastapi.tiangolo.com/tutorial/path-params/",
"https://fastapi.tiangolo.com/tutorial/query-params/"
]
print("🚀 Learning FastAPI from official documentation...")
fastapi_info = learn_library_from_urls("FastAPI", fastapi_urls)
# Example 2: Learn a different library (you can replace with any library)
streamlit_urls = [
"https://docs.streamlit.io/",
"https://docs.streamlit.io/get-started",
"https://docs.streamlit.io/develop/api-reference"
]
print("\n\n📊 Learning Streamlit from official documentation...")
streamlit_info = learn_library_from_urls("Streamlit", streamlit_urls)
FastAPI 공식 문서 4개, Streamlit 문서 3개를 넘겨 학습을 시켜요. 어떤 라이브러리든 URL만 바꾸면 돼요.
Step 3: 코드 예시 생성하기
def generate_examples_for_library(library_info: Dict, library_name: str):
"""Generate code examples for any library based on its documentation."""
# Define generic use cases that can apply to most libraries
use_cases = [
{
"name": "Basic Setup and Hello World",
"description": f"Create a minimal working example with {library_name}",
"requirements": "Include installation, imports, and basic usage"
},
{
"name": "Common Operations",
"description": f"Demonstrate the most common {library_name} operations",
"requirements": "Show typical workflow and best practices"
},
{
"name": "Advanced Usage",
"description": f"Create a more complex example showcasing {library_name} capabilities",
"requirements": "Include error handling and optimization"
}
]
generated_examples = []
print(f"\n🔧 Generating examples for {library_name}...")
for use_case in use_cases:
print(f"\n📝 {use_case['name']}")
print(f"Description: {use_case['description']}")
example = agent.generate_example(
library_info=library_info,
use_case=use_case['description'],
requirements=use_case['requirements']
)
print("\n💻 Generated Code:")
print("```python")
print(example['code'])
print("```")
print("\n📦 Required Imports:")
for imp in example['imports']:
print(f" • {imp}")
print("\n📝 Explanation:")
print(example['explanation'])
print("\n✅ Best Practices:")
for practice in example['best_practices']:
print(f" • {practice}")
generated_examples.append({
"use_case": use_case['name'],
"code": example['code'],
"imports": example['imports'],
"explanation": example['explanation'],
"best_practices": example['best_practices']
})
print("-" * 80)
return generated_examples
# Generate examples for both libraries
print("🎯 Generating FastAPI Examples:")
fastapi_examples = generate_examples_for_library(fastapi_info, "FastAPI")
print("\n\n🎯 Generating Streamlit Examples:")
streamlit_examples = generate_examples_for_library(streamlit_info, "Streamlit")
기본 설정, 공통 동작, 고급 사용이라는 세 가지 범용 사용 사례에 대해 generate_example을 호출해 코드와 설명, 모범 사례를 만들어 내요.
Step 4: 대화형 라이브러리 학습 함수
def learn_any_library(library_name: str, documentation_urls: list[str], use_cases: list[str] = None):
"""Learn any library from its documentation and generate examples."""
if use_cases is None:
use_cases = [
"Basic setup and hello world example",
"Common operations and workflows",
"Advanced usage with best practices"
]
print(f"🚀 Starting automated learning for {library_name}...")
print(f"Documentation sources: {len(documentation_urls)} URLs")
try:
# Step 1: Learn from documentation
library_info = agent.learn_from_urls(library_name, documentation_urls)
# Step 2: Generate examples for each use case
all_examples = []
for i, use_case in enumerate(use_cases, 1):
print(f"\n📝 Generating example {i}/{len(use_cases)}: {use_case}")
example = agent.generate_example(
library_info=library_info,
use_case=use_case,
requirements="Include error handling, comments, and follow best practices"
)
all_examples.append({
"use_case": use_case,
"code": example['code'],
"imports": example['imports'],
"explanation": example['explanation'],
"best_practices": example['best_practices']
})
return {
"library_info": library_info,
"examples": all_examples
}
except Exception as e:
print(f"❌ Error learning {library_name}: {e}")
return None
def interactive_learning_session():
"""Interactive session for learning libraries with user input."""
print("🎯 Welcome to the Interactive Library Learning System!")
print("This system will help you learn any Python library from its documentation.\n")
learned_libraries = {}
while True:
print("\n" + "="*60)
print("🚀 LIBRARY LEARNING SESSION")
print("="*60)
# Get library name from user
library_name = input("\n📚 Enter the library name you want to learn (or 'quit' to exit): ").strip()
if library_name.lower() in ['quit', 'exit', 'q']:
print("\n👋 Thanks for using the Interactive Library Learning System!")
break
if not library_name:
print("❌ Please enter a valid library name.")
continue
# Get documentation URLs
print(f"\n🔗 Enter documentation URLs for {library_name} (one per line, empty line to finish):")
urls = []
while True:
url = input(" URL: ").strip()
if not url:
break
if not url.startswith(('http://', 'https://')):
print(" ⚠️ Please enter a valid URL starting with http:// or https://")
continue
urls.append(url)
if not urls:
print("❌ No valid URLs provided. Skipping this library.")
continue
# Get custom use cases from user
print(f"\n🎯 Define use cases for {library_name} (optional, press Enter for defaults):")
print(" Default use cases will be: Basic setup, Common operations, Advanced usage")
user_wants_custom = input(" Do you want to define custom use cases? (y/n): ").strip().lower()
use_cases = None
if user_wants_custom in ['y', 'yes']:
print(" Enter your use cases (one per line, empty line to finish):")
use_cases = []
while True:
use_case = input(" Use case: ").strip()
if not use_case:
break
use_cases.append(use_case)
if not use_cases:
print(" No custom use cases provided, using defaults.")
use_cases = None
# Learn the library
print(f"\n🚀 Starting learning process for {library_name}...")
result = learn_any_library(library_name, urls, use_cases)
if result:
learned_libraries[library_name] = result
print(f"\n✅ Successfully learned {library_name}!")
# Show summary
print(f"\n📊 Learning Summary for {library_name}:")
print(f" • Core concepts: {len(result['library_info']['core_concepts'])} identified")
print(f" • Common patterns: {len(result['library_info']['patterns'])} found")
print(f" • Examples generated: {len(result['examples'])}")
# Ask if user wants to see examples
show_examples = input(f"\n👀 Do you want to see the generated examples for {library_name}? (y/n): ").strip().lower()
if show_examples in ['y', 'yes']:
for i, example in enumerate(result['examples'], 1):
print(f"\n{'─'*50}")
print(f"📝 Example {i}: {example['use_case']}")
print(f"{'─'*50}")
print("\n💻 Generated Code:")
print("```python")
print(example['code'])
print("```")
print(f"\n📦 Required Imports:")
for imp in example['imports']:
print(f" • {imp}")
print(f"\n📝 Explanation:")
print(example['explanation'])
print(f"\n✅ Best Practices:")
for practice in example['best_practices']:
print(f" • {practice}")
# Ask if user wants to see the next example
if i < len(result['examples']):
continue_viewing = input(f"\nContinue to next example? (y/n): ").strip().lower()
if continue_viewing not in ['y', 'yes']:
break
# Offer to save results
save_results = input(f"\n💾 Save learning results for {library_name} to file? (y/n): ").strip().lower()
if save_results in ['y', 'yes']:
filename = input(f" Enter filename (default: {library_name.lower()}_learning.json): ").strip()
if not filename:
filename = f"{library_name.lower()}_learning.json"
try:
import json
with open(filename, 'w') as f:
json.dump(result, f, indent=2, default=str)
print(f" ✅ Results saved to {filename}")
except Exception as e:
print(f" ❌ Error saving file: {e}")
else:
print(f"❌ Failed to learn {library_name}")
# Ask if user wants to learn another library
print(f"\n📚 Libraries learned so far: {list(learned_libraries.keys())}")
continue_learning = input("\n🔄 Do you want to learn another library? (y/n): ").strip().lower()
if continue_learning not in ['y', 'yes']:
break
# Final summary
if learned_libraries:
print(f"\n🎉 Session Summary:")
print(f"Successfully learned {len(learned_libraries)} libraries:")
for lib_name, info in learned_libraries.items():
print(f" • {lib_name}: {len(info['examples'])} examples generated")
return learned_libraries
# Example: Run interactive learning session
if __name__ == "__main__":
# Run interactive session
learned_libraries = interactive_learning_session()
interactive_learning_session은 사용자에게 라이브러리 이름과 문서 URL, 커스텀 사용 사례를 입력받아 학습을 진행해요. 사용 사례를 직접 정의할 수도 있고 기본값(기본 설정, 공통 동작, 고급 사용)을 쓸 수도 있어요. 결과는 JSON 파일로 저장할 수 있어요.
예시 출력
FastAPI에 대해 대화형 세션을 실행하면 대략 이런 흐름이에요.
- 세션이 시작되고 라이브러리 이름으로
FastAPI를 입력 - 문서 URL 3개를 입력 (예:
https://fastapi.tiangolo.com/,.../tutorial/first-steps/,.../tutorial/path-params/) - 커스텀 사용 사례로 "REST API with authentication", "file upload endpoint", "SQLAlchemy database integration"을 정의
- 문서 처리:
Core Concepts: ['FastAPI app', 'path operations', 'dependencies', 'request/response models'],Common Patterns: ['app = FastAPI()', 'decorator-based routing', 'Pydantic models'],Key Methods: ['FastAPI()', '@app.get()', '@app.post()', 'uvicorn.run()']같은 결과가 나와요 - 코드 생성: 인증이 있는 REST API 예시를 만들고, JWT 기반 로그인 엔드포인트와 보호된 라우트를 포함한 코드, 필요한 import, 설명, 모범 사례(비밀키는 환경변수 사용, 프로덕션에서 비밀번호 해싱 등)를 보여줘요
- 결과를
fastapi_learning.json으로 저장
다음 단계
- GitHub 통합: README와 예시 저장소에서 학습
- 비디오 튜토리얼 처리: 영상 문서에서 정보 추출
- 커뮤니티 예시: Stack Overflow와 포럼의 예시 집계
- 버전 비교: 라이브러리 버전 간 API 변화 추적
- 테스트 생성: 생성된 코드에 대한 단위 테스트 자동 작성
- 페이지 크롤링: 문서 페이지를 자동 크롤링해 사용법을 적극적으로 이해
이 튜토리얼은 DSPy가 낯선 라이브러리를 문서에서 학습하는 전체 과정을 자동화하는 모습을 보여줘요. 빠른 기술 도입과 탐색에 특히 유용해요.
더 알아보기 (Learn more)
- ChainOfThought 모듈 — 분석·생성 단계를 구성하는 모듈
- Signature 소개 — 클래스 기반 signature 작성법