Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ The command-line interface provides commands for downloading, deploying, and del
<a id="cli-download"></a>
### Download

The `download` command retrieves Databus files, versions, artifacts, groups, collections, or SPARQL query results. It supports authentication, checksum validation, compression conversion, and RDF or tabular format conversion. See the [download docs](doc/cli-usage.md#cli-download).
The `download` command retrieves Databus files, versions, artifacts, groups, accounts, collections, or SPARQL query results. It supports authentication, checksum validation, compression conversion, and RDF or tabular format conversion. See the [download docs](doc/cli-usage.md#cli-download).

<a id="cli-deploy"></a>
### Deploy
Expand Down
142 changes: 140 additions & 2 deletions databusclient/api/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -1419,6 +1419,126 @@ def _get_databus_artifacts_of_group(json_str: str) -> List[str]:
return result


def _get_groups_of_account(
account_uri: str,
endpoint_url: str,
databus_key: str | None = None,
) -> List[str]:
"""Query a Databus SPARQL endpoint to find all groups belonging to an account.

Args:
account_uri: The full databus account URI (e.g. 'https://databus.dbpedia.org/dbpedia').
endpoint_url: The Databus SPARQL endpoint URL.
databus_key: Optional Databus API key for authentication on protected databus.

Returns:
List of group URIs belonging to the account.
"""
clean_account = (
"https://"
+ account_uri.removeprefix("http://").removeprefix("https://").rstrip("/")
)
query_v2 = f"""PREFIX databus: <https://dataid.dbpedia.org/databus#>
SELECT DISTINCT ?group WHERE {{
?dataset databus:account <{clean_account}> ;
databus:group ?group .
}} ORDER BY ?group"""

try:
result = _query_sparql_endpoint(endpoint_url, query_v2, databus_key=databus_key)
bindings = result.get("results", {}).get("bindings", [])
except Exception as e:
print(f"Warning: SPARQL query for account groups failed: {e}")
bindings = []

if not bindings:
query_legacy = f"""PREFIX dataid: <http://dataid.dbpedia.org/ns/core#>
SELECT DISTINCT ?group WHERE {{
?dataset dataid:account <{clean_account}> ;
dataid:group ?group .
}} ORDER BY ?group"""
result = _query_sparql_endpoint(endpoint_url, query_legacy, databus_key=databus_key)
bindings = result.get("results", {}).get("bindings", [])

group_uris: List[str] = []
for binding in bindings:
group_val = binding.get("group", {}).get("value")
if isinstance(group_val, str) and group_val.startswith(f"{clean_account}/"):
if group_val not in group_uris:
group_uris.append(group_val)

return group_uris


def _download_account(
uri: str,
endpoint: str,
localDir: str,
all_versions: bool = False,
vault_token_file: str = None,
databus_key: str = None,
auth_url: str = None,
client_id: str = None,
compression: str = None,
convert_format: str = None,
graph_name: str = None,
base_uri: str = None,
graph_mode: str = None,
manifest_context=None,
validate_checksum: bool = False,
user_agent: str | None = None,
) -> None:
"""Download all groups, artifacts and versions of an account from Databus.

Args:
uri: The full databus account URI.
endpoint: The databus SPARQL endpoint URL.
localDir: Local directory to download files to. If None, the databus folder structure is created in the current working directory.
all_versions: If True, download all versions of each artifact in the account; otherwise, only download the latest version.
vault_token_file: Path to Vault refresh token file for protected downloads.
databus_key: Databus API key for protected downloads.
auth_url: Keycloak token endpoint URL.
client_id: Client ID for token exchange.
compression: Target compression format for on-the-fly conversion.
convert_format: Target RDF/tabular format for on-the-fly conversion.
graph_name: Named graph URI for Triple -> Quad conversion (Layer 3).
base_uri: Base URI for CSV -> Triple conversion (Layer 3).
graph_mode: Optional graph sidecar mode. Currently supports 'download-url'.
validate_checksum: Whether to validate checksums after downloading.
user_agent: HTTP User-Agent for file HEAD and GET requests. None uses
the versioned databus-python-client default.
"""
clean_account = uri.rstrip("/")
group_uris = _get_groups_of_account(clean_account, endpoint, databus_key=databus_key)
if not group_uris:
print(f"No groups found for account: {clean_account}")
return

print(f"Found {len(group_uris)} group(s) for account {clean_account}:")
for g in group_uris:
print(f" - {g}")

for group_uri in group_uris:
print(f"Downloading group: {group_uri}")
_download_group(
group_uri,
localDir,
all_versions=all_versions,
vault_token_file=vault_token_file,
databus_key=databus_key,
auth_url=auth_url,
client_id=client_id,
compression=compression,
convert_format=convert_format,
graph_name=graph_name,
base_uri=base_uri,
graph_mode=graph_mode,
manifest_context=manifest_context,
validate_checksum=validate_checksum,
user_agent=user_agent,
)


def download(
localDir: str,
endpoint: str,
Expand All @@ -1439,7 +1559,7 @@ def download(
) -> None:
"""Download datasets from databus.

Download of files, versions, artifacts, groups or databus collections via their databus URIs or user-defined SPARQL queries that return file download URLs.
Download of files, versions, artifacts, groups, accounts or databus collections via their databus URIs or user-defined SPARQL queries that return file download URLs.

Args:
localDir: Local directory to download datasets to. If None, the databus folder structure is created in the current working directory.
Expand Down Expand Up @@ -1580,7 +1700,25 @@ def download(
user_agent=user_agent,
)
elif account is not None:
print("accountId not supported yet") # TODO
print(f"Downloading account and all its groups: {databusURI}")
_download_account(
databusURI,
uri_endpoint,
localDir,
all_versions=all_versions,
vault_token_file=token,
databus_key=databus_key,
auth_url=auth_url,
client_id=client_id,
compression=compression,
convert_format=convert_format,
graph_name=graph_name,
base_uri=base_uri,
graph_mode=graph_mode,
manifest_context=manifest_context,
validate_checksum=validate_checksum,
user_agent=user_agent,
)
else:
print(
"dataId not supported yet"
Expand Down
7 changes: 6 additions & 1 deletion doc/cli-usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ databusclient [delete|deploy|download|manifest|workflow] --help
<a id="cli-download"></a>
### Download

With the download command, you can download datasets or parts thereof from the Databus. The download command expects one or more Databus URIs or a SPARQL query as arguments. The URIs can point to files, versions, artifacts, groups, or collections. If a SPARQL query is provided, the query must return download URLs from the Databus which will be downloaded.
With the download command, you can download datasets or parts thereof from the Databus. The download command expects one or more Databus URIs or a SPARQL query as arguments. The URIs can point to files, versions, artifacts, groups, accounts, or collections. If a SPARQL query is provided, the query must return download URLs from the Databus which will be downloaded.

```bash
databusclient download $DOWNLOADTARGET
Expand Down Expand Up @@ -65,6 +65,11 @@ databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased
databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals
```

**Download Account**: download of all files with the latest version of all artifacts across all groups owned by an account
```bash
databusclient download https://databus.dbpedia.org/dbpedia
```

**Download Group**: download of all files with the latest version of all artifacts of a group
```bash
databusclient download https://databus.dbpedia.org/dbpedia/mappings
Expand Down
Loading