Direct Memory Access to Files in Collections

1. Overview

The guide HTTPS Access to Collections describes how to directly access files in a Globus Collection for upload and download.

This guide builds on that foundation by showing how applications can access collection data directly from memory, eliminating the need to first transfer data to or from local files.

Applications access collections using the standard HTTPS protocol, allowing them to use widely available tools and libraries—across many programming languages—to read and write data directly between a collection and application memory using HTTP GET and PUT requests.

2. HTTPS Base URL

The first step is to find the HTTP address of your collection. You can do this with the following command:

globus gcs collection show "6c54cade-bde5-45c1-bdea-f4bd71dba2cc"

In the command output, look for the https_url field. Its value will look similar to: https://m-d3a2c3.collection1.tutorials.globus.org.

This value is the base URL for your collection. Any file within the collection can be accessed by appending its path to this base URL. For example, if the collection contains the file: /sandbox/hello.txt then the full HTTP URL to that file would be: https://m-d3a2c3.collection1.tutorials.globus.org/sandbox/hello.txt.

3. Bearer Token

In order to access a collection over HTTPS, you must prove who you are (authentication) and that you are allowed to access the data (authorization). This is done by requesting a Bearer Token from Globus Auth.

3.1. Registering an Application

The first thing needed is a Client ID. This is a UUID that uniquely identifies your application to Globus Auth and tells the service which application is requesting access. Instructions for registering an application and obtaining a Client ID are available here.

3.2. Requesting A Token

Once you have a Client ID, the following Python code will guide you through the process of obtaining a bearer token for a specific collection.

import globus_sdk


collection_id = "f8f77d69-1c12-4de8-bac5-3822bef999af"
client_id = "5defd655-241e-44fd-8d3b-89b6efd12920"


required_scopes = {
    collection_id: [
        f"https://auth.globus.org/scopes/{collection_id}/https",
        f"https://auth.globus.org/scopes/{collection_id}/data_access",
    ],
    "transfer.api.globus.org": [
        "urn:globus:auth:scope:transfer.api.globus.org:all"
    ],
}

app = globus_sdk.UserApp(
    "my-app",
    config=globus_sdk.GlobusAppConfig(
        environment="test",
        auto_redrive_gares=True,
    ),
    client_id=client_id,
    scope_requirements=required_scopes,
)

# Start the OAuth login flow
app.login()

# Retrieve the token for the collection
token = app.token_storage.get_token_data(collection_id)

print(token.access_token)

While this may seem like quite a bit of code, on closer examination, it is simple. The basic flow is: 1. Set the scopes required for direct access 2. Create a basic application and login 3. Extract the token and print it.

This code connects to Globus Auth and requests an access token for the specified scopes. During execution, it will guide you through the OAuth2 login process so you can authenticate and authorize the application.

Once the flow is complete, the program prints the bearer token that can be used in HTTPS requests to the collection.

Note

A bearer token is a secret credential. Printing it to the console, as shown here, is only done for demonstration purposes. In production code, tokens should be stored and handled securely.

3.3. Accessing Data Directly from Memory

Once you have the HTTP address of the file you want and a valid access token, you can read and write files directly to/from your application’s memory, provided your account has the appropriate permissions for the target path.

3.3.1. Reading Data from Collections

Here are a few examples in various programming languages. Notice that there is no Globus-specific code at all. All you need to do is set the Authorization header with your bearer token, and you can read remote collection data directly into your application’s memory space.

import urllib.request


url = "https://m-d3a2c3.collection1.tutorials.globus.org/sandbox/hello.txt"
token = "YOUR BEARER TOKEN"


req = urllib.request.Request(
   url,
   headers={"Authorization": f"Bearer {token}"}
)


block_size = 1024  # bytes per block


with urllib.request.urlopen(req) as resp:
   while True:
       chunk = resp.read(block_size)
       if not chunk:
           break
       # Process the chunk; here we just print it
       print(chunk.decode(), end="")

package main

import (
    "fmt"
    "io"
    "net/http"
)

func main() {
    url := "https://m-d3a2c3.collection1.tutorials.globus.org/sandbox/hello.txt"
    token := "YOUR BEARER TOKEN"

    req, err := http.NewRequest("GET", url, nil)
    if err != nil {
        panic(err)
    }

    req.Header.Set("Authorization", "Bearer "+token)

    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close()

    blockSize := 1024
    buf := make([]byte, blockSize)

    for {
        n, err := resp.Body.Read(buf)
        if n > 0 {
            fmt.Print(string(buf[:n]))
        }
        if err != nil {
            if err == io.EOF {
                break
            }
            panic(err)
        }
    }
}

import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;

public class Main {
    public static void main(String[] args) {
        try {
            String urlStr = "https://m-d3a2c3.collection1.tutorials.globus.org/sandbox/hello.txt";
            String token = "YOUR BEARER TOKEN";

            URL url = new URL(urlStr);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();

            conn.setRequestMethod("GET");
            conn.setRequestProperty("Authorization", "Bearer " + token);

            InputStream in = conn.getInputStream();

            int blockSize = 1024;
            byte[] buffer = new byte[blockSize];
            int bytesRead;

            while ((bytesRead = in.read(buffer)) != -1) {
                System.out.print(new String(buffer, 0, bytesRead));
            }

            in.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

3.3.2. Writing Data to Collections

You can write data from memory to a collection using HTTP PUT requests. The same Authorization header is used for write operations.

import urllib.request


url = "https://m-d3a2c3.collection1.tutorials.globus.org/sandbox/output.txt"
token = "YOUR BEARER TOKEN"

# Data to write (generated in memory)
data = "Hello from memory!\n".encode('utf-8')


req = urllib.request.Request(
    url,
    data=data,
    method='PUT',
    headers={
        "Authorization": f"Bearer {token}",
        "Content-Type": "text/plain"
    }
)


with urllib.request.urlopen(req) as resp:
    print(f"Upload status: {resp.status}")

package main

import (
    "bytes"
    "fmt"
    "net/http"
)

func main() {
    url := "https://m-d3a2c3.collection1.tutorials.globus.org/sandbox/output.txt"
    token := "YOUR BEARER TOKEN"

    // Data to write (generated in memory)
    data := []byte("Hello from memory!\n")

    req, err := http.NewRequest("PUT", url, bytes.NewReader(data))
    if err != nil {
        panic(err)
    }

    req.Header.Set("Authorization", "Bearer "+token)
    req.Header.Set("Content-Type", "text/plain")

    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close()

    fmt.Printf("Upload status: %d\n", resp.StatusCode)
}

import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;

public class Main {
    public static void main(String[] args) {
        try {
            String urlStr = "https://m-d3a2c3.collection1.tutorials.globus.org/sandbox/output.txt";
            String token = "YOUR BEARER TOKEN";

            // Data to write (generated in memory)
            byte[] data = "Hello from memory!\n".getBytes();

            URL url = new URL(urlStr);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();

            conn.setRequestMethod("PUT");
            conn.setRequestProperty("Authorization", "Bearer " + token);
            conn.setRequestProperty("Content-Type", "text/plain");
            conn.setDoOutput(true);

            OutputStream out = conn.getOutputStream();
            out.write(data);
            out.close();

            System.out.println("Upload status: " + conn.getResponseCode());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}