Merge pull request #1146 from khushthecoder/GH1056-local-file-urls-escaped-paths

Fix #1056: Decode URL-escaped paths for local filesystems

Author: jonbrenas

malariagen_data/util.py

@@ -494,6 +494,25 @@ def _init_filesystem(url, **kwargs):
     # Process the URL using fsspec.
     fs, path = url_to_fs(url, **storage_options)
 
+    # Decode URL-encoded paths for local filesystems.
+    protocol = getattr(fs, "protocol", None)
+    if isinstance(protocol, str):
+        protocols = {protocol}
+    elif isinstance(protocol, (tuple, list)):
+        protocols = set(protocol)
+    else:
+        protocols = set()
+
+    is_local = (
+        bool(protocols.intersection({"file", "local"}))
+        or fs.__class__.__name__ == "LocalFileSystem"
+    )
+
+    if is_local:
+        from urllib.parse import unquote
+
+        path = unquote(path)
+
     # Path compatibility, fsspec/gcsfs behaviour varies between versions.
     while path.endswith("/"):
         path = path[:-1]

tests/test_util_filesystem.py

@@ -0,0 +1,35 @@
+import os
+from malariagen_data.util import _init_filesystem
+
+
+def test_init_filesystem_decodes_file_uri_escaped_path(tmp_path):
+    dir_with_space = tmp_path / "dir with space's"
+    dir_with_space.mkdir()
+    file_path = dir_with_space / "v3-config.json"
+    file_path.write_text('{"foo": "bar"}')
+
+    uri = dir_with_space.as_uri()
+
+    fs, path = _init_filesystem(uri)
+
+    assert "%20" in uri
+    assert "%20" not in path
+    assert "%27" not in path
+
+    # Using local path with os.path.join should now succeed
+    with fs.open(os.path.join(path, "v3-config.json"), "r") as f:
+        assert f.read() == '{"foo": "bar"}'
+
+
+def test_init_filesystem_plain_local_path_unchanged(tmp_path):
+    dir_with_space = tmp_path / "dir with space's"
+    dir_with_space.mkdir()
+    file_path = dir_with_space / "v3-config.json"
+    file_path.write_text('{"foo": "bar"}')
+
+    uri = str(dir_with_space)
+
+    fs, path = _init_filesystem(uri)
+
+    with fs.open(os.path.join(path, "v3-config.json"), "r") as f:
+        assert f.read() == '{"foo": "bar"}'