🔄 Storage Transfer Service

# Google Cloud Consoleでの手順 1. Cloud Console → Storage Transfer Service 2. [転送ジョブを作成] 3. ソースの選択: - Amazon S3 bucket - Bucket name: my-s3-bucket - AWS Access Key ID: [入力] - AWS Secret Access Key: [入力] 4. 宛先の選択: - GCS bucket: gs://my-destination-bucket/ - Path: data/imported/ 5. オプション設定: - ☑ 転送後にソースからファイルを削除 - ☑ 既存ファイルを上書き - フィルタ: prefix = "logs/" 6. スケジュール: - 毎日午前2時に実行 7. [作成]

# AWS S3からGCSへの転送ジョブを作成 gcloud transfer jobs create \ s3://my-s3-bucket/data/ \ gs://my-gcs-bucket/imported-data/ \ --source-creds-file=aws-credentials.json \ --schedule-starts="2024-11-26T02:00:00Z" \ --schedule-repeats-every="1d" \ --delete-from-source-after-transfer \ --overwrite-when="different" # GCS間の転送ジョブを作成 gcloud transfer jobs create \ gs://source-bucket/path/ \ gs://destination-bucket/path/ \ --schedule-starts="2024-11-26T00:00:00Z" \ --schedule-repeats-every="7d" # 転送ジョブの一覧表示 gcloud transfer jobs list # 転送オペレーションの確認 gcloud transfer operations list \ --job-names="transferJobs/12345678901234567890"

from google.cloud import storage_transfer def create_s3_transfer_job( project_id: str, s3_bucket: str, gcs_bucket: str, aws_access_key_id: str, aws_secret_access_key: str ): client = storage_transfer.StorageTransferServiceClient() # 転送ジョブの設定 transfer_job = { "project_id": project_id, "description": "S3 to GCS daily transfer", "status": "ENABLED", "schedule": { "schedule_start_date": { "year": 2024, "month": 11, "day": 26 }, "start_time_of_day": { "hours": 2, "minutes": 0 } }, "transfer_spec": { "aws_s3_data_source": { "bucket_name": s3_bucket, "aws_access_key": { "access_key_id": aws_access_key_id, "secret_access_key": aws_secret_access_key } }, "gcs_data_sink": { "bucket_name": gcs_bucket, "path": "imported/" }, "object_conditions": { "include_prefixes": ["logs/"] }, "transfer_options": { "delete_objects_from_source_after_transfer": False, "overwrite_objects_already_existing_in_sink": True } } } result = client.create_transfer_job(request={"transfer_job": transfer_job}) print(f"Created transfer job: {result.name}") return result