검색 및 분석 엔진 시작하기
검색 및 분석 엔진 시작하기
이 코드 예제에서는 Amazon OpenSearch Service를 활용해 도메인 생성 → 도메인에 데이터 업로드 → 검색 → 리소스 정리까지를 Bash 스크립트로 보여줍니다. IAM 자격 증명이 설정된 환경에서 OpenSearch 도메인을 만들고 데이터를 넣어 기본 검색·분석 엔진 운영을 체험할 수 있어요.
출처: 문서
본문
#!/bin/bash
# Amazon OpenSearch Service Getting Started Script - Version 8 Fixed
# This script creates an OpenSearch domain, uploads data, searches documents, and cleans up resources
# Based on the tested and working 4-tutorial-final.md
# SECURITY IMPROVEMENTS IN THIS VERSION:
# 1. Removed hardcoded passwords - now generated securely
# 2. Improved access policy to use principle of least privilege
# 3. Added credential masking in logs
# 4. Removed credentials from command output
# 5. Added input validation for AWS region
# 6. Improved error handling and validation
# 7. Added secure temporary file handling
# 8. Removed unnecessary curl insecurity flags
# 9. Added better secret management practices
# 10. Improved resource tagging for better governance
set -e # Exit on any error
# Set up logging with restricted permissions
LOG_FILE="opensearch_tutorial_v8_fixed.log"
touch "$LOG_FILE"
chmod 600 "$LOG_FILE"
exec > >(tee -a "$LOG_FILE") 2>&1
echo "Starting Amazon OpenSearch Service tutorial script v8-fixed at $(date)"
echo "All commands and outputs will be logged to $LOG_FILE"
# Track if domain was successfully created
DOMAIN_CREATED=false
DOMAIN_ACTIVE=false
# Secure temporary directory
TEMP_DIR=$(mktemp -d)
trap 'rm -rf "$TEMP_DIR"' EXIT
chmod 700 "$TEMP_DIR"
# Error handling function
handle_error() {
echo "ERROR: $1"
echo "Attempting to clean up resources..."
cleanup_resources
exit 1
}
# Function to clean up resources
cleanup_resources() {
echo "Cleaning up resources..."
if [[ "$DOMAIN_CREATED" == "true" ]]; then
echo "Checking if domain $DOMAIN_NAME exists before attempting to delete..."
# Check if domain exists before trying to delete
if aws opensearch describe-domain --domain-name "$DOMAIN_NAME" > /dev/null 2>&1; then
echo "Domain $DOMAIN_NAME exists. Proceeding with deletion."
aws opensearch delete-domain --domain-name "$DOMAIN_NAME"
echo "Domain deletion initiated. This may take several minutes to complete."
else
echo "Domain $DOMAIN_NAME does not exist or is not accessible. No deletion needed."
fi
else
echo "No domain was successfully created. Nothing to clean up."
fi
# Securely clean up sensitive files
if [[ -d "$TEMP_DIR" ]]; then
shred -vfz -n 3 "$TEMP_DIR"/* 2>/dev/null || true
rm -rf "$TEMP_DIR"
fi
}
# Generate a random identifier for resource names to avoid conflicts
RANDOM_ID=$(openssl rand -hex 4)
DOMAIN_NAME="movies-${RANDOM_ID}"
MASTER_USER="master-user"
# Generate secure password using openssl instead of hardcoding
MASTER_PASSWORD=$(openssl rand -base64 32 | tr -d "=+/" | cut -c1-25)
# Mask password in logs
echo "Using domain name: $DOMAIN_NAME"
echo "Using master username: $MASTER_USER"
echo "Using master password: [REDACTED]"
# Get AWS account ID (matches tutorial)
echo "Retrieving AWS account ID..."
ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
if [[ $? -ne 0 ]] || [[ -z "$ACCOUNT_ID" ]]; then
handle_error "Failed to retrieve AWS account ID. Please check your AWS credentials."
fi
echo "AWS Account ID: $ACCOUNT_ID"
# Get current region (matches tutorial)
echo "Retrieving current AWS region..."
AWS_REGION=$(aws configure get region)
if [[ -z "$AWS_REGION" ]]; then
AWS_REGION="us-east-1"
echo "No region found in AWS config, defaulting to $AWS_REGION"
else
echo "Using AWS region: $AWS_REGION"
fi
# Validate AWS region format
if ! [[ "$AWS_REGION" =~ ^[a-z]{2}-[a-z]+-[0-9]$ ]]; then
handle_error "Invalid AWS region format: $AWS_REGION"
fi
# Step 1: Create an OpenSearch Service Domain
echo "Creating OpenSearch Service domain..."
echo "This may take 15-30 minutes to complete."
# SECURITY IMPROVED: Use least privilege access policy
# This policy restricts access to specific actions and should be further restricted to specific principals in production
# Create the domain using CloudFormation (handles the 15-20 min creation wait)
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
STACK_NAME="tutorial-opensearch-${RANDOM_ID}"
echo "Creating domain $DOMAIN_NAME via CloudFormation..."
echo "This typically takes 15-20 minutes. CloudFormation will wait for completion."
aws cloudformation deploy \
--template-file "$SCRIPT_DIR/cfn-opensearch-domain.yaml" \
--stack-name "$STACK_NAME" \
--parameter-overrides \
DomainName="$DOMAIN_NAME" \
MasterUserName="$MASTER_USER" \
MasterUserPassword="$MASTER_PASSWORD" \
--tags project=doc-smith tutorial=opensearch-service-gs \
--no-fail-on-empty-changeset 2>&1 || handle_error "CloudFormation stack creation failed"
DOMAIN_CREATED=true
echo "Domain created successfully via CloudFormation."
# Get domain endpoint from stack outputs
DOMAIN_ENDPOINT=$(aws cloudformation describe-stacks \
--stack-name "$STACK_NAME" \
--query 'Stacks[0].Outputs[?OutputKey==`DomainEndpoint`].OutputValue' \
--output text)
if [[ -z "$DOMAIN_ENDPOINT" ]] || [[ "$DOMAIN_ENDPOINT" == "None" ]]; then
handle_error "Failed to get domain endpoint from CloudFormation outputs"
fi
echo "Domain endpoint: $DOMAIN_ENDPOINT"
# Wait for fine-grained access control to be fully ready
echo "Waiting for fine-grained access control to initialize..."
sleep 120
# Verify variables are set correctly (matches tutorial)
echo "Verifying configuration..."
echo "Domain endpoint: $DOMAIN_ENDPOINT"
echo "Master user: $MASTER_USER"
echo "Password set: $(if [ -n "$MASTER_PASSWORD" ]; then echo "Yes"; else echo "No"; fi)"
# Step 2: Upload Data to the Domain
echo "Preparing to upload data to the domain..."
# Create a file for the single document (matches tutorial exactly)
echo "Creating single document JSON file..."
SINGLE_MOVIE_FILE="$TEMP_DIR/single_movie.json"
cat > "$SINGLE_MOVIE_FILE" << 'EOF'
{
"director": "Burton, Tim",
"genre": ["Comedy","Sci-Fi"],
"year": 1996,
"actor": ["Jack Nicholson","Pierce Brosnan","Sarah Jessica Parker"],
"title": "Mars Attacks!"
}
EOF
chmod 600 "$SINGLE_MOVIE_FILE"
# Create a file for bulk documents (matches tutorial exactly)
echo "Creating bulk documents JSON file..."
BULK_MOVIES_FILE="$TEMP_DIR/bulk_movies.json"
cat > "$BULK_MOVIES_FILE" << 'EOF'
{ "index" : { "_index": "movies", "_id" : "2" } }
{"director": "Frankenheimer, John", "genre": ["Drama", "Mystery", "Thriller", "Crime"], "year": 1962, "actor": ["Lansbury, Angela", "Sinatra, Frank", "Leigh, Janet", "Harvey, Laurence", "Silva, Henry", "Frees, Paul", "Gregory, James", "Bissell, Whit", "McGiver, John", "Parrish, Leslie", "Edwards, James", "Flowers, Bess", "Dhiegh, Khigh", "Payne, Julie", "Kleeb, Helen", "Gray, Joe", "Nalder, Reggie", "Stevens, Bert", "Masters, Michael", "Lowell, Tom"], "title": "The Manchurian Candidate"}
{ "index" : { "_index": "movies", "_id" : "3" } }
{"director": "Baird, Stuart", "genre": ["Action", "Crime", "Thriller"], "year": 1998, "actor": ["Downey Jr., Robert", "Jones, Tommy Lee", "Snipes, Wesley", "Pantoliano, Joe", "Jacob, Irène", "Nelligan, Kate", "Roebuck, Daniel", "Malahide, Patrick", "Richardson, LaTanya", "Wood, Tom", "Kosik, Thomas", "Stellate, Nick", "Minkoff, Robert", "Brown, Spitfire", "Foster, Reese", "Spielbauer, Bruce", "Mukherji, Kevin", "Cray, Ed", "Fordham, David", "Jett, Charlie"], "title": "U.S. Marshals"}
{ "index" : { "_index": "movies", "_id" : "4" } }
{"director": "Ray, Nicholas", "genre": ["Drama", "Romance"], "year": 1955, "actor": ["Hopper, Dennis", "Wood, Natalie", "Dean, James", "Mineo, Sal", "Backus, Jim", "Platt, Edward", "Ray, Nicholas", "Hopper, William", "Allen, Corey", "Birch, Paul", "Hudson, Rochelle", "Doran, Ann", "Hicks, Chuck", "Leigh, Nelson", "Williams, Robert", "Wessel, Dick", "Bryar, Paul", "Sessions, Almira", "McMahon, David", "Peters Jr., House"], "title": "Rebel Without a Cause"}
EOF
chmod 600 "$BULK_MOVIES_FILE"
# Check if curl is installed
if ! command -v curl &> /dev/null; then
echo "Warning: curl is not installed. Skipping data upload and search steps."
echo "You can manually upload the data later using the commands in the tutorial."
else
# IMPROVED: Test authentication with multiple approaches
echo "Testing authentication with the OpenSearch domain..."
echo "This test checks if fine-grained access control is ready for data operations."
# Create credentials file for secure handling
CREDENTIALS_FILE="$TEMP_DIR/.credentials"
echo "$MASTER_USER:$MASTER_PASSWORD" > "$CREDENTIALS_FILE"
chmod 600 "$CREDENTIALS_FILE"
# Test 1: Basic authentication with root endpoint
echo "Testing basic authentication with root endpoint..."
AUTH_TEST_RESULT=$(curl -s -w "\nHTTP_CODE:%{http_code}" \
--netrc-file "$CREDENTIALS_FILE" \
--request GET \
"https://${DOMAIN_ENDPOINT}/" 2>&1)
echo "Basic auth test result:"
echo "$AUTH_TEST_RESULT"
# Extract HTTP status code
HTTP_CODE=$(echo "$AUTH_TEST_RESULT" | grep "HTTP_CODE:" | cut -d: -f2)
# Function to check if HTTP code is 2xx
is_success_code() {
local code=$1
if [[ "$code" =~ ^2[0-9][0-9]$ ]]; then
return 0
else
return 1
fi
}
# Check if basic authentication test was successful (200 or 2xx status codes)
if is_success_code "$HTTP_CODE"; then
echo "â Basic authentication test successful! (HTTP $HTTP_CODE)"
AUTH_SUCCESS=true
AUTH_METHOD="basic"
else
echo "Basic authentication failed with HTTP code: $HTTP_CODE"
# Test 2: Try cluster health endpoint
echo "Testing with cluster health endpoint..."
HEALTH_TEST_RESULT=$(curl -s -w "\nHTTP_CODE:%{http_code}" \
--netrc-file "$CREDENTIALS_FILE" \
--request GET \
"https://${DOMAIN_ENDPOINT}/_cluster/health" 2>&1)
echo "Cluster health test result:"
echo "$HEALTH_TEST_RESULT"
HEALTH_HTTP_CODE=$(echo "$HEALTH_TEST_RESULT" | grep "HTTP_CODE:" | cut -d: -f2)
if is_success_code "$HEALTH_HTTP_CODE"; then
echo "â Cluster health authentication test successful! (HTTP $HEALTH_HTTP_CODE)"
AUTH_SUCCESS=true
AUTH_METHOD="basic"
else
echo "Cluster health authentication also failed with HTTP code: $HEALTH_HTTP_CODE"
# Check for specific error patterns
if echo "$AUTH_TEST_RESULT" | grep -q "anonymous is not authorized"; then
echo "Error: Request is being treated as anonymous (authentication not working)"
elif echo "$AUTH_TEST_RESULT" | grep -q "Unauthorized"; then
echo "Error: Authentication credentials rejected"
elif echo "$AUTH_TEST_RESULT" | grep -q "Forbidden"; then
echo "Error: Authentication succeeded but access is forbidden"
fi
echo "Waiting additional time and retrying with exponential backoff..."
# Retry authentication test with exponential backoff
AUTH_RETRY_COUNT=0
MAX_AUTH_RETRIES=5
WAIT_TIME=60
AUTH_SUCCESS=false
while [[ $AUTH_RETRY_COUNT -lt $MAX_AUTH_RETRIES ]]; do
echo "Retrying authentication test (attempt $((AUTH_RETRY_COUNT+1))/$MAX_AUTH_RETRIES) after ${WAIT_TIME} seconds..."
sleep $WAIT_TIME
# Try both endpoints
AUTH_TEST_RESULT=$(curl -s -w "\nHTTP_CODE:%{http_code}" \
--netrc-file "$CREDENTIALS_FILE" \
--request GET \
"https://${DOMAIN_ENDPOINT}/" 2>&1)
HTTP_CODE=$(echo "$AUTH_TEST_RESULT" | grep "HTTP_CODE:" | cut -d: -f2)
echo "Retry result (HTTP $HTTP_CODE):"
echo "$AUTH_TEST_RESULT"
if is_success_code "$HTTP_CODE"; then
echo "â Authentication test successful after retry! (HTTP $HTTP_CODE)"
AUTH_SUCCESS=true
AUTH_METHOD="basic"
break
fi
# Also try cluster health
HEALTH_TEST_RESULT=$(curl -s -w "\nHTTP_CODE:%{http_code}" \
--netrc-file "$CREDENTIALS_FILE" \
--request GET \
"https://${DOMAIN_ENDPOINT}/_cluster/health" 2>&1)
HEALTH_HTTP_CODE=$(echo "$HEALTH_TEST_RESULT" | grep "HTTP_CODE:" | cut -d: -f2)
if is_success_code "$HEALTH_HTTP_CODE"; then
echo "â Cluster health authentication successful after retry! (HTTP $HEALTH_HTTP_CODE)"
AUTH_SUCCESS=true
AUTH_METHOD="basic"
break
fi
AUTH_RETRY_COUNT=$((AUTH_RETRY_COUNT+1))
# Exponential backoff: double the wait time each retry (max 10 minutes)
WAIT_TIME=$((WAIT_TIME * 2))
if [[ $WAIT_TIME -gt 600 ]]; then
WAIT_TIME=600
fi
done
fi
fi
# Proceed with data operations if authentication is working
if [[ "$AUTH_SUCCESS" == "true" ]]; then
echo "Authentication successful using $AUTH_METHOD method. Proceeding with data operations."
# Upload single document (matches tutorial exactly)
echo "Uploading single document..."
UPLOAD_RESULT=$(curl -s -w "\nHTTP_CODE:%{http_code}" \
--netrc-file "$CREDENTIALS_FILE" \
--request PUT \
--header 'Content-Type: application/json' \
--data @"$SINGLE_MOVIE_FILE" \
"https://${DOMAIN_ENDPOINT}/movies/_doc/1" 2>&1)
echo "Upload response:"
echo "$UPLOAD_RESULT"
UPLOAD_HTTP_CODE=$(echo "$UPLOAD_RESULT" | grep "HTTP_CODE:" | cut -d: -f2)
if is_success_code "$UPLOAD_HTTP_CODE" && echo "$UPLOAD_RESULT" | grep -q '"result"'; then
echo "â Single document uploaded successfully! (HTTP $UPLOAD_HTTP_CODE)"
else
echo "â Warning: Single document upload may have failed (HTTP $UPLOAD_HTTP_CODE)"
fi
# Upload bulk documents (matches tutorial exactly)
echo "Uploading bulk documents..."
BULK_RESULT=$(curl -s -w "\nHTTP_CODE:%{http_code}" \
--netrc-file "$CREDENTIALS_FILE" \
--request POST \
--header 'Content-Type: application/x-ndjson' \
--data-binary @"$BULK_MOVIES_FILE" \
"https://${DOMAIN_ENDPOINT}/movies/_bulk" 2>&1)
echo "Bulk upload response:"
echo "$BULK_RESULT"
BULK_HTTP_CODE=$(echo "$BULK_RESULT" | grep "HTTP_CODE:" | cut -d: -f2)
if is_success_code "$BULK_HTTP_CODE" && echo "$BULK_RESULT" | grep -q '"errors": false'; then
echo "â Bulk documents uploaded successfully! (HTTP $BULK_HTTP_CODE)"
else
echo "â Warning: Bulk document upload may have failed (HTTP $BULK_HTTP_CODE)"
fi
# Wait a moment for indexing
echo "Waiting for documents to be indexed..."
sleep 5
# Step 3: Search Documents (matches tutorial exactly)
echo "Searching for documents containing 'mars'..."
SEARCH_RESULT=$(curl -s -w "\nHTTP_CODE:%{http_code}" \
--netrc-file "$CREDENTIALS_FILE" \
--request GET \
"https://${DOMAIN_ENDPOINT}/movies/_search?q=mars&pretty=true" 2>&1)
SEARCH_HTTP_CODE=$(echo "$SEARCH_RESULT" | grep "HTTP_CODE:" | cut -d: -f2)
echo "Search results for 'mars' (HTTP $SEARCH_HTTP_CODE):"
echo "$SEARCH_RESULT"
echo "Searching for documents containing 'rebel'..."
REBEL_SEARCH=$(curl -s -w "\nHTTP_CODE:%{http_code}" \
--netrc-file "$CREDENTIALS_FILE" \
--request GET \
"https://${DOMAIN_ENDPOINT}/movies/_search?q=rebel&pretty=true" 2>&1)
REBEL_HTTP_CODE=$(echo "$REBEL_SEARCH" | grep "HTTP_CODE:" | cut -d: -f2)
echo "Search results for 'rebel' (HTTP $REBEL_HTTP_CODE):"
echo "$REBEL_SEARCH"
# Verify search results
if is_success_code "$SEARCH_HTTP_CODE" && echo "$SEARCH_RESULT" | grep -q '"hits"'; then
echo "â Search functionality is working!"
else
echo "â Warning: Search may not be working properly."
fi
else
echo ""
echo "=========================================="
echo "AUTHENTICATION TROUBLESHOOTING"
echo "=========================================="
echo "Authentication failed after all retries. This may be due to:"
echo "1. Fine-grained access control not fully initialized (most common)"
echo "2. Domain configuration issues"
echo "3. Network connectivity issues"
echo "4. AWS credentials or permissions issues"
echo ""
echo "DOMAIN CONFIGURATION DEBUG:"
echo "Let's check the domain configuration..."
# Debug domain configuration
DOMAIN_CONFIG=$(aws opensearch describe-domain --domain-name "$DOMAIN_NAME" --query 'DomainStatus.{AdvancedSecurityOptions: AdvancedSecurityOptions, AccessPolicies: AccessPolicies}' --output json 2>&1)
echo "Domain configuration:"
echo "$DOMAIN_CONFIG"
echo ""
echo "MANUAL TESTING COMMANDS:"
echo "You can try these commands manually in 10-15 minutes:"
echo ""
echo "# Test basic authentication:"
echo "curl --user \"${MASTER_USER}:[PASSWORD]\" \"https://${DOMAIN_ENDPOINT}/\""
echo ""
echo "# Test cluster health:"
echo "curl --user \"${MASTER_USER}:[PASSWORD]\" \"https://${DOMAIN_ENDPOINT}/_cluster/health\""
echo ""
echo "# Upload single document:"
echo "curl --user \"${MASTER_USER}:[PASSWORD]\" --request PUT --header 'Content-Type: application/json' --data @single_movie.json \"https://${DOMAIN_ENDPOINT}/movies/_doc/1\""
echo ""
echo "# Search for documents:"
echo "curl --user \"${MASTER_USER}:[PASSWORD]\" \"https://${DOMAIN_ENDPOINT}/movies/_search?q=mars&pretty=true\""
echo ""
echo "TROUBLESHOOTING TIPS:"
echo "- Wait 10-15 more minutes and try the manual commands"
echo "- Check AWS CloudTrail logs for authentication errors"
echo "- Verify your AWS region is correct: $AWS_REGION"
echo "- Ensure your AWS credentials have OpenSearch permissions"
echo "- Try accessing OpenSearch Dashboards to verify the master user works"
echo ""
echo "Skipping data upload and search operations for now."
echo "The domain is created and accessible via OpenSearch Dashboards."
fi
# Securely remove credentials file
shred -vfz -n 3 "$CREDENTIALS_FILE" 2>/dev/null || true
fi
# Display OpenSearch Dashboards URL (matches tutorial)
echo ""
echo "==========================================="
echo "OPENSEARCH DASHBOARDS ACCESS"
echo "==========================================="
echo "OpenSearch Dashboards URL: https://${DOMAIN_ENDPOINT}/_dashboards/"
echo "Username: $MASTER_USER"
echo "Password: [REDACTED - stored securely]"
echo ""
echo "You can access OpenSearch Dashboards using the credentials provided during domain creation."
echo "If you uploaded data successfully, you can create an index pattern for 'movies'."
echo ""
# Summary of created resources
echo ""
echo "==========================================="
echo "RESOURCES CREATED"
echo "==========================================="
echo "OpenSearch Domain Name: $DOMAIN_NAME"
echo "OpenSearch Domain Endpoint: $DOMAIN_ENDPOINT"
echo "AWS Region: $AWS_REGION"
echo "Master Username: $MASTER_USER"
echo ""
echo "ESTIMATED COST: ~$0.038/hour (~$0.91/day) until deleted"
echo ""
echo "Make sure to save the domain name and endpoint for future reference."
echo ""
# Auto-confirm cleanup: Yes, clean up all resources
echo ""
echo "==========================================="
echo "CLEANUP CONFIRMATION"
echo "==========================================="
echo "Automatically cleaning up all created resources..."
CLEANUP_CHOICE="y"
if [[ "${CLEANUP_CHOICE,,}" == "y" ]]; then
echo "Cleaning up resources..."
if [ -n "${STACK_NAME:-}" ]; then
echo "Deleting CloudFormation stack $STACK_NAME..."
aws cloudformation delete-stack --stack-name "$STACK_NAME"
echo "â Stack deletion initiated. This may take several minutes."
echo " Monitor: aws cloudformation describe-stacks --stack-name $STACK_NAME"
else
aws opensearch delete-domain --domain-name "$DOMAIN_NAME" 2>/dev/null
echo "â Domain deletion initiated."
fi
else
echo "Resources will NOT be deleted automatically."
echo ""
echo "To delete later:"
echo " aws cloudformation delete-stack --stack-name ${STACK_NAME:-tutorial-opensearch-XXXX}"
echo ""
echo "â IMPORTANT: Keeping these resources will incur ongoing AWS charges!"
fi
# Clean up temporary files (handled by trap)
echo "Cleaning up temporary files..."
# Disable the trap since we're handling cleanup manually
trap - EXIT
# Final cleanup
rm -rf "$TEMP_DIR" 2>/dev/null || true
echo ""
echo "==========================================="
echo "SCRIPT COMPLETED SUCCESSFULLY"
echo "==========================================="
echo "Script completed at $(date)"
echo "All output has been logged to: $LOG_FILE"
echo ""
echo "Next steps:"
echo "1. Access OpenSearch Dashboards at: https://${DOMAIN_ENDPOINT}/_dashboards/"
echo "2. Create visualizations and dashboards"
echo "3. Explore the OpenSearch API"
echo "4. Remember to delete resources when done to avoid charges"
참고: 이 예제의 전체 소스와 실행 방법은 GitHub의 샘플 개발자 튜토리얼 저장소에서 확인할 수 있어요.
더 알아보기 (Learn more)
관련 API 동작을 더 자세히 보려면 AWS CLI 명령 참조 문서를 확인하세요.