#!/usr/bin/python
# ***** Program to perform various CLI utility functions for Amazon S3 Access
# ***** Written by Gary Morris
import boto3
import argparse
import json
import hashlib
from botocore.client import Config
from pprint import pprint
# Default bucket and region
bucket_name = "vps-testbucket"
region = "us-east-1"
# Function to create client connection object
def client_connect():
s3client = boto3.client('s3', region)
return s3client
def delete_objects(key):
client = client_connect()
result = client.delete_objects(Bucket=bucket_name, Delete={'Objects': [
{ 'Key': key }
] })
pprint(result)
# Function to return a pre-signed URL given a key. This is a pre-configured stub for now
def get_presigned_url():
client = client_connect()
url = client.generate_presigned_url (
ClientMethod='get_object',
Params={
'Bucket': bucket_name,
'Key': '063e5e33ba13a97b2f9fe1ae8be524f6-121/20170101/bill10.pdf'
}
)
print url
# List objects given a Prefix
def get_objects(key):
client = client_connect()
response = client.list_objects(Bucket=bucket_name, Prefix=key)
for content in response["Contents"]:
print content["Key"]
# List buckets
def list_buckets():
s3 = boto3.resource('s3')
s3client = boto3.client('s3', region)
response = s3client.list_buckets()
for bucket in response["Buckets"]:
print(bucket['Name'])
# for key in bucket.objects.all():
# print(key.key)
# Upload a JSON-serialized file of bill data. customer_id is stubbed and for future should add for validation to prevent overwriting a customer accidentally
def upload_file(filename,customer_id):
with open(filename) as data_file:
data = json.load(data_file)
# pprint(data)
client = client_connect()
for customer in data["customers"]:
## Upload file in format s3.upload_file ("tmp.txt", "bucket-name", "key-name",
## ExtraArgs={"Metadata": {"mykey": "myvalue"}} )
#
# Key is created using hash of customer_id-client_account_number
hashval=hashlib.md5(customer["id"] + customer["client_account_number"]).hexdigest() + "-" + customer["id"]
client.upload_file(customer["filename"],
bucket_name,
hashval + "/" + customer["statement_date"] + "/" + customer["filename"],
ExtraArgs={"Metadata": {"customer_id": customer["id"], "current_amount": customer["current_amount"], "client_account_number": customer["client_account_number"]}} )
## Include tagging, which is required to get bill information for display or filtering
response = client.put_object_tagging(
Bucket=bucket_name,
Key=hashval + "/" + customer["statement_date"] + "/" + customer["filename"],
Tagging={
'TagSet': [
{
'Key': 'customer_id',
'Value': customer["id"]
},
{
'Key': 'current_amount',
'Value': customer["current_amount"]
},
{
'Key': 'client_account_number',
'Value': customer["client_account_number"]
},
{
'Key': 'statement_date',
'Value': customer["statement_date"]
}
]
}
)
pprint(response)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='AWS S3 API Utility')
parser.add_argument('--list', action='store_true', help="List Buckets")
parser.add_argument('--upload-file', dest='upload_file', help="Upload a file.")
parser.add_argument('--customer', dest='customer_id', help="Customer ID")
parser.add_argument('--get-url', dest='get_url', action='store_true', help="Get Presigned URL")
parser.add_argument('--get-objects', dest='get_objects', action='store_true', help="List Objects, requires --key")
parser.add_argument('--delete-objects', dest="delete_objects", action='store_true', help="Delete objets, requires --key")
parser.add_argument('--key', dest='key', help='Lookup Key')
args = parser.parse_args()
if (args.list):
list_buckets()
elif (args.upload_file):
upload_file(args.upload_file,args.customer_id)
elif (args.get_url):
get_presigned_url()
elif (args.get_objects):
get_objects(args.key)
elif (args.delete_objects):
delete_objects(args.key)
else:
print parser.print_help()