Module Installation

Connection to server

Using a config file for credentials

import swiftclient
import json
with open("../config.json") as json_data_file:
    config = json.load(json_data_file)
#print(config['swift_storage']['user'])

def swift_con(config):
    user=config['swift_storage']['user']
    key=config['swift_storage']['key']
    auth_url=config['swift_storage']['auth_url']
    tenant_name=config['swift_storage']['tenant_name']
    auth_version=config['swift_storage']['auth_version']
    options = config['swift_storage']['options']
    return swiftclient.Connection(user=user,
                                  key=key,
                                  authurl=auth_url,
                                  os_options=options,
                                  tenant_name=tenant_name,
                                  auth_version=auth_version)

conn = swift_con(config)

List files

for container in conn.get_account()[1]:
    container_name = container['name']
    print('container_name:', container_name)
objects = conn.get_container(container_name)[1]
for data in objects:
    if 'oco2_1504' in data['name']:
        print('{0}\t{1}\t{2}'.format(data['name'], data['bytes'], data['last_modified']))

Upload files

import glob
import os
from tqdm import tqdm_notebook as tqdm

def upload_to_swift(mask, prefix,content_type):
    for file in tqdm(glob.glob(mask, recursive=True)):
        with open(file, 'rb') as one_file:
                upload_to = prefix+ os.path.basename(file)
                #print('Copy from',file,'to',upload_to)
                conn.put_object(container_name, upload_to,
                                                contents= one_file.read(),
                                                content_type=content_type) # 'text/csv'

upload_to_swift("/media/data-nvme/dev/datasets/OCO2/csv/*.csv", "/datasets/oco-2/peaks-detected/", 'text/csv')
upload_to_swift("/media/data-nvme/dev/datasets/OCO2/csv/*.json", "/datasets/oco-2/peaks-detected-details/", 'application/json')

Upload HTML

upload_to_swift("chemin/peaks_and_sources.html", "/map/", 'text/html')

Copy from /home/ben/Downloads/peaks_and_sources.html to /map/peaks_and_sources.html

Delete files

container_name = 'oco2'
for data in tqdm(conn.get_container(container_name)[1]):
    file = data['name']
    if 'peaks-detected' in file:
        #print('delete', file)
        # UNCOMMENT FOR REAL DELETE conn.delete_object(container_name, file)