save and load dictionary to s3 using bz compression
full docs here https://thanakijwanavit.github.io/s3bz/
pip install s3bz
First, import the s3 module
from importlib import reload
from s3bz.s3bz import S3
result = S3.save(key = key,
objectToSave = sampleDict,
bucket = bucket,
user=USER,
pw = PW,
accelerate = True)
print(('failed', 'success')[result])
result = S3.load(key = key,
bucket = bucket,
user = USER,
pw = PW,
accelerate = True)
print(result[0])
print(bucket)
%time S3.saveZl(key,sampleDict,bucket)
%time S3.loadZl(key,bucket)
%time S3.savePklZl(key,sampleDict,bucket)
%time result =S3.loadPklZl(key,bucket)
import gzip, json
compressor=lambda x: gzip.compress(x)
encoder=lambda x: json.dumps(x).encode()
decompressor=lambda x: gzip.decompress(x)
decoder=lambda x: json.loads(x.decode())
%time S3.generalSave(key, sampleDict, bucket = bucket, compressor=compressor, encoder=encoder )
%time result = S3.generalLoad(key, bucket , decompressor=decompressor, decoder=decoder)
assert result == sampleDict, 'not the same as sample dict'
result = S3.exist('', bucket, user=USER, pw=PW, accelerate = True)
print(('doesnt exist', 'exist')[result])
url = S3.presign(key=key,
bucket=bucket,
expiry = 1000,
user=USER,
pw=PW)
print(url)
from s3bz.s3bz import Requests
result = Requests.getContentFromUrl(url)
inputPath = '/tmp/tmpFile.txt'
key = 'tmpFile'
downloadPath = '/tmp/downloadTmpFile.txt'
with open(inputPath , 'w')as f:
f.write('hello world')
S3.saveFile(key =key ,path = inputPath,bucket = bucket)
##test
S3.exist(key,bucket)
S3.loadFile(key= key , path = downloadPath, bucket = bucket)
with open(downloadPath, 'r') as f:
print(f.read())
result = S3.deleteFile(key, bucket)
## test
S3.exist(key,bucket)
### this is not include in the requirements to minimize the size impact
import pandas as pd
df = pd.DataFrame({'test':[1,2,3,4,5],'test2':[2,3,4,5,6]})
S3.saveDataFrame(bucket,key,df)
S3.loadDataFrame(bucket,key)
from s3bz.s3bz import ExtraArgs, S3
bucket = 'pybz-test'
key = 'test.dict'
fields = {**ExtraArgs.jpeg}
S3.presignUpload(bucket, key, fields=fields)