Skip to content

sync

sync

Registry generation from the Zenodo API (fwl-io sync).

For every dataset in a manifest, query the pinned Zenodo record and rewrite the committed registry file with the record's file names and checksums. This is the only place where hashes enter the repository, so registry changes are always visible in review.

Concept DOIs are rejected here: a concept DOI resolves to the newest deposit of a record (the API answers with a redirect to the latest version, so the returned record id differs from the requested one), which would let the data change underneath pinned code. Only version DOIs are accepted.

All datasets of a manifest are attempted; per-dataset failures are collected and raised together at the end, so one bad entry does not leave the rest of the registry set unwritten.

fetch_zenodo_record(doi, api_base=ZENODO_API)

Return the full Zenodo API record for a pinned version DOI.

A concept DOI is rejected: the API resolves it to the newest deposit (the returned record id differs from the requested one, or equals the concept record id), which would let the data change underneath pinned code.

Source code in src/fwl_io/sync.py
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
def fetch_zenodo_record(doi: str, api_base: str = ZENODO_API) -> dict:
    """Return the full Zenodo API record for a pinned version DOI.

    A concept DOI is rejected: the API resolves it to the newest deposit (the
    returned record id differs from the requested one, or equals the concept
    record id), which would let the data change underneath pinned code.
    """
    recid = zenodo_record_id(doi)
    require_online(f'reading Zenodo record {recid}')
    response = requests.get(f'{api_base}/{recid}', timeout=30)
    response.raise_for_status()
    record = response.json()

    returned_id = str(record.get('id'))
    if returned_id != recid or str(record.get('conceptrecid')) == recid:
        raise ValueError(
            f'{doi} is a concept DOI (the API resolves it to the newest deposit, '
            f'record {returned_id}); pin the version DOI of a specific deposit instead'
        )
    return record

fetch_zenodo_registry(doi, api_base=ZENODO_API)

Return the name-to-checksum mapping of a pinned Zenodo record.

Source code in src/fwl_io/sync.py
88
89
90
91
92
93
94
def fetch_zenodo_registry(doi: str, api_base: str = ZENODO_API) -> dict[str, str]:
    """Return the name-to-checksum mapping of a pinned Zenodo record."""
    record = fetch_zenodo_record(doi, api_base=api_base)
    entries = _extract_files(record)
    if not entries:
        raise ValueError(f'Zenodo record {zenodo_record_id(doi)} lists no files')
    return entries

in_community(record)

Whether a Zenodo record, as :func:fetch_zenodo_record returns it, is an accepted record of one of :data:ZENODO_COMMUNITIES (listed in metadata.communities). An entry of another form is no match.

Source code in src/fwl_io/sync.py
77
78
79
80
81
82
83
84
85
def in_community(record: dict) -> bool:
    """Whether a Zenodo record, as :func:`fetch_zenodo_record` returns it, is an accepted
    record of one of :data:`ZENODO_COMMUNITIES` (listed in ``metadata.communities``). An
    entry of another form is no match."""
    metadata = record.get('metadata') if isinstance(record, dict) else None
    listed = metadata.get('communities') if isinstance(metadata, dict) else None
    return isinstance(listed, list) and any(
        isinstance(entry, dict) and entry.get('id') in ZENODO_COMMUNITIES for entry in listed
    )

select_files(entries, wanted, *, source)

Return the entries named in wanted; raise when one is not there.

A name absent from the record is an error rather than a silent omission, so a misspelt or removed file cannot leave a dataset quietly incomplete.

Source code in src/fwl_io/sync.py
 97
 98
 99
100
101
102
103
104
105
106
107
108
def select_files(
    entries: dict[str, str], wanted: tuple[str, ...] | list[str], *, source: str
) -> dict[str, str]:
    """Return the entries named in ``wanted``; raise when one is not there.

    A name absent from the record is an error rather than a silent omission, so
    a misspelt or removed file cannot leave a dataset quietly incomplete.
    """
    absent = sorted(set(wanted) - set(entries))
    if absent:
        raise ValueError(f'{source} does not contain {absent}; it lists {sorted(entries)}')
    return {name: entries[name] for name in wanted}

sync_dataset(dataset, api_base=ZENODO_API)

Regenerate the committed registry file for one dataset.

Source code in src/fwl_io/sync.py
111
112
113
114
115
116
117
118
119
120
121
def sync_dataset(dataset: Dataset, api_base: str = ZENODO_API) -> Path:
    """Regenerate the committed registry file for one dataset."""
    if not dataset.zenodo:
        raise ValueError(f'dataset {dataset.key!r} has no Zenodo DOI; cannot sync')
    if dataset.registry_path is None:
        raise ValueError(f'dataset {dataset.key!r} has no registry path')
    entries = fetch_zenodo_registry(dataset.zenodo, api_base=api_base)
    if dataset.files is not None:
        entries = select_files(entries, dataset.files, source=f'Zenodo record of {dataset.key!r}')
    write_registry(dataset.registry_path, entries)
    return dataset.registry_path

sync_manifest(manifest_path, api_base=ZENODO_API)

Regenerate the registries of every dataset in a manifest.

Every dataset is attempted; if any fail, the successful registries are still written and a single aggregated error is raised at the end.

Raises:

Type Description
OfflineModeError

If FWL_IO_OFFLINE is set.

RuntimeError

If any dataset fails to sync.

Source code in src/fwl_io/sync.py
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
def sync_manifest(manifest_path: str | Path, api_base: str = ZENODO_API) -> list[Path]:
    """Regenerate the registries of every dataset in a manifest.

    Every dataset is attempted; if any fail, the successful registries are
    still written and a single aggregated error is raised at the end.

    Raises
    ------
    OfflineModeError
        If ``FWL_IO_OFFLINE`` is set.
    RuntimeError
        If any dataset fails to sync.
    """
    require_online(f'syncing {manifest_path}')
    written: list[Path] = []
    failures: dict[str, str] = {}
    for ds in load_manifest(manifest_path):
        try:
            written.append(sync_dataset(ds, api_base=api_base))
        except Exception as exc:  # noqa: BLE001 -- aggregate and re-raise below
            failures[ds.key] = str(exc)
    if failures:
        detail = '\n'.join(f'  {key}: {msg}' for key, msg in sorted(failures.items()))
        raise RuntimeError(
            f'{len(failures)} dataset(s) failed to sync '
            f'({len(written)} registries written):\n{detail}'
        )
    return written

zenodo_record_id(doi)

Return the numeric record id of a Zenodo version DOI.

Parameters:

Name Type Description Default
doi str

A Zenodo DOI of the form 10.5281/zenodo.<record-id> (an optional doi: prefix is accepted).

required

Returns:

Type Description
str

The trailing record-id digits.

Raises:

Type Description
ValueError

When the string is not a Zenodo DOI.

Source code in src/fwl_io/doi.py
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
def zenodo_record_id(doi: str) -> str:
    """Return the numeric record id of a Zenodo version DOI.

    Parameters
    ----------
    doi : str
        A Zenodo DOI of the form ``10.5281/zenodo.<record-id>`` (an optional
        ``doi:`` prefix is accepted).

    Returns
    -------
    str
        The trailing record-id digits.

    Raises
    ------
    ValueError
        When the string is not a Zenodo DOI.
    """
    match = ZENODO_DOI_PATTERN.fullmatch(doi.strip())
    if not match:
        raise ValueError(f'{doi!r} is not a Zenodo DOI of the form 10.5281/zenodo.<id>')
    return match.group(2)