diff --git a/.gitignore b/.gitignore index 73102171..70a5ae65 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +*.dylib +*.dylib.yml +Info.plist *.so *.whl diff --git a/Cargo.lock b/Cargo.lock index 41a276b3..2601b3f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1520,9 +1520,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf85e27e86080aafd5a22eae58a162e133a589551542b3e5cee4beb27e54f8e1" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ "chrono", "chrono-tz", @@ -1572,9 +1572,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ "python3-dll-a", "target-lexicon", @@ -1592,9 +1592,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "491aa5fc66d8059dd44a75f4580a2962c1862a1c2945359db36f6c2818b748dc" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", @@ -1611,9 +1611,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d671734e9d7a43449f8480f8b38115df67bef8d21f76837fa75ee7aaa5e52e" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1623,9 +1623,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22faaa1ce6c430a1f71658760497291065e6450d7b5dc2bcf254d49f66ee700a" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", diff --git a/obstore/src/copy.rs b/obstore/src/copy.rs index 4e3a9f8e..277d6ef6 100644 --- a/obstore/src/copy.rs +++ b/obstore/src/copy.rs @@ -3,7 +3,7 @@ use std::{future::Future, pin::Pin}; use object_store::ObjectStoreExt; use pyo3::prelude::*; use pyo3_async_runtimes::tokio::get_runtime; -use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult}; +use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult, PyPath}; use crate::utils::PyNone; @@ -12,8 +12,8 @@ use crate::utils::PyNone; pub(crate) fn copy( py: Python, store: PyObjectStore, - from_: String, - to: String, + from_: PyPath, + to: PyPath, overwrite: bool, ) -> PyObjectStoreResult<()> { let runtime = get_runtime(); @@ -35,8 +35,8 @@ pub(crate) fn copy( pub(crate) fn copy_async( py: Python, store: PyObjectStore, - from_: String, - to: String, + from_: PyPath, + to: PyPath, overwrite: bool, ) -> PyResult> { let from_ = from_.into(); diff --git a/obstore/src/get.rs b/obstore/src/get.rs index b5d5f51f..31c25914 100644 --- a/obstore/src/get.rs +++ b/obstore/src/get.rs @@ -346,7 +346,7 @@ pub(crate) fn get( ) -> PyObjectStoreResult { let runtime = get_runtime(); py.detach(|| { - let path = &path.as_ref(); + let path = path.as_ref(); let fut = if let Some(options) = options { store.as_ref().get_opts(path, options.into()) } else { @@ -366,7 +366,7 @@ pub(crate) fn get_async( options: Option, ) -> PyResult> { pyo3_async_runtimes::tokio::future_into_py(py, async move { - let path = &path.as_ref(); + let path = path.as_ref(); let fut = if let Some(options) = options { store.as_ref().get_opts(path, options.into()) } else { diff --git a/obstore/src/list.rs b/obstore/src/list.rs index 9ba9568c..253787c2 100644 --- a/obstore/src/list.rs +++ b/obstore/src/list.rs @@ -8,7 +8,6 @@ use arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use futures::stream::{BoxStream, Fuse}; use futures::StreamExt; use indexmap::IndexMap; -use object_store::path::Path; use object_store::{ListResult, ObjectMeta, ObjectStore}; use pyo3::exceptions::{PyImportError, PyStopAsyncIteration, PyStopIteration}; use pyo3::prelude::*; @@ -17,7 +16,7 @@ use pyo3::{intern, IntoPyObjectExt}; use pyo3_arrow::export::{Arro3RecordBatch, Arro3Table}; use pyo3_arrow::PyTable; use pyo3_async_runtimes::tokio::get_runtime; -use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult}; +use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult, PyPath}; use tokio::sync::Mutex; pub(crate) struct PyObjectMeta(ObjectMeta); @@ -353,8 +352,8 @@ impl<'py> IntoPyObject<'py> for PyListResult { pub(crate) fn list( py: Python, store: PyObjectStore, - prefix: Option, - offset: Option, + prefix: Option, + offset: Option, chunk_size: usize, return_arrow: bool, ) -> PyObjectStoreResult { @@ -373,7 +372,7 @@ pub(crate) fn list( let store = store.into_inner().clone(); let prefix = prefix.map(|s| s.into()); let stream = if let Some(offset) = offset { - store.list_with_offset(prefix.as_ref(), &offset.into()) + store.list_with_offset(prefix.as_ref(), offset.as_ref()) } else { store.list(prefix.as_ref()) }; @@ -385,14 +384,14 @@ pub(crate) fn list( pub(crate) fn list_with_delimiter( py: Python, store: PyObjectStore, - prefix: Option, + prefix: Option, return_arrow: bool, ) -> PyObjectStoreResult { let runtime = get_runtime(); py.detach(|| { let out = runtime.block_on(list_with_delimiter_materialize( store.into_inner(), - prefix.map(|s| s.into()).as_ref(), + prefix.as_ref(), return_arrow, ))?; Ok::<_, PyObjectStoreError>(out) @@ -404,25 +403,24 @@ pub(crate) fn list_with_delimiter( pub(crate) fn list_with_delimiter_async( py: Python, store: PyObjectStore, - prefix: Option, + prefix: Option, return_arrow: bool, ) -> PyResult> { pyo3_async_runtimes::tokio::future_into_py(py, async move { - let out = list_with_delimiter_materialize( - store.into_inner(), - prefix.map(|s| s.into()).as_ref(), - return_arrow, - ) - .await?; + let out = + list_with_delimiter_materialize(store.into_inner(), prefix.as_ref(), return_arrow) + .await?; Ok(out) }) } async fn list_with_delimiter_materialize( store: Arc, - prefix: Option<&Path>, + prefix: Option<&PyPath>, return_arrow: bool, ) -> PyObjectStoreResult { - let list_result = store.list_with_delimiter(prefix).await?; + let list_result = store + .list_with_delimiter(prefix.map(|s| s.as_ref())) + .await?; Ok(PyListResult::new(list_result, return_arrow)) } diff --git a/obstore/src/rename.rs b/obstore/src/rename.rs index 8a9e04d4..3003ed75 100644 --- a/obstore/src/rename.rs +++ b/obstore/src/rename.rs @@ -3,7 +3,7 @@ use std::{future::Future, pin::Pin}; use object_store::ObjectStoreExt; use pyo3::prelude::*; use pyo3_async_runtimes::tokio::get_runtime; -use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult}; +use pyo3_object_store::{PyObjectStore, PyObjectStoreError, PyObjectStoreResult, PyPath}; use crate::utils::PyNone; @@ -12,8 +12,8 @@ use crate::utils::PyNone; pub(crate) fn rename( py: Python, store: PyObjectStore, - from_: String, - to: String, + from_: PyPath, + to: PyPath, overwrite: bool, ) -> PyObjectStoreResult<()> { let runtime = get_runtime(); @@ -35,8 +35,8 @@ pub(crate) fn rename( pub(crate) fn rename_async( py: Python, store: PyObjectStore, - from_: String, - to: String, + from_: PyPath, + to: PyPath, overwrite: bool, ) -> PyResult> { let from_ = from_.into(); diff --git a/tests/test_copy.py b/tests/test_copy.py new file mode 100644 index 00000000..c16d85ff --- /dev/null +++ b/tests/test_copy.py @@ -0,0 +1,23 @@ +from obstore.store import MemoryStore + + +def test_copy_non_ascii(): + store = MemoryStore() + + name1 = "café.txt" + name2 = "ümlaut.txt" + name3 = "こんにちは世界.txt" + name4 = "你好世界.txt" + + store.put(name1, b"foo") + store.put(name3, b"bar") + + store.copy(name1, name2) + store.copy(name3, name4) + + result = store.list().collect() + assert len(result) == 4 + assert result[0]["path"] == name1 + assert result[1]["path"] == name2 + assert result[2]["path"] == name3 + assert result[3]["path"] == name4 diff --git a/tests/test_head.py b/tests/test_head.py new file mode 100644 index 00000000..fffc5b54 --- /dev/null +++ b/tests/test_head.py @@ -0,0 +1,20 @@ +from obstore.store import MemoryStore + + +def test_head_non_ascii(): + store = MemoryStore() + + name1 = "café.txt" + name2 = "ümlaut.txt" + name3 = "こんにちは世界.txt" + name4 = "你好世界.txt" + + store.put(name1, b"foo") + store.put(name2, b"bar") + store.put(name3, b"baz") + store.put(name4, b"qux") + + assert store.head(name1)["path"] == name1 + assert store.head(name2)["path"] == name2 + assert store.head(name3)["path"] == name3 + assert store.head(name4)["path"] == name4 diff --git a/tests/test_list.py b/tests/test_list.py index 3030605e..dcec45a2 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -11,11 +11,27 @@ def test_list(): store.put("file2.txt", b"bar") store.put("file3.txt", b"baz") - stream = store.list() - result = stream.collect() + result = store.list().collect() assert len(result) == 3 +def test_list_non_ascii(): + store = MemoryStore() + + name1 = "café.txt" + name2 = "ümlaut.txt" + name3 = "こんにちは世界.txt" + store.put(name1, b"foo") + store.put(name2, b"bar") + store.put(name3, b"baz") + + result = store.list().collect() + assert len(result) == 3 + assert result[0]["path"] == name1 + assert result[1]["path"] == name2 + assert result[2]["path"] == name3 + + def test_list_as_arrow(): store = MemoryStore() @@ -37,6 +53,23 @@ def test_list_as_arrow(): assert batch.num_rows == 100 +def test_list_non_ascii_arrow(): + store = MemoryStore() + + name1 = "café.txt" + name2 = "ümlaut.txt" + name3 = "こんにちは世界.txt" + store.put(name1, b"foo") + store.put(name2, b"bar") + store.put(name3, b"baz") + + result = store.list(return_arrow=True).collect() + assert result.num_rows == 3 + assert result["path"][0].as_py() == name1 + assert result["path"][1].as_py() == name2 + assert result["path"][2].as_py() == name3 + + @pytest.mark.asyncio async def test_list_stream_async(): store = MemoryStore() diff --git a/tests/test_rename.py b/tests/test_rename.py new file mode 100644 index 00000000..a84111c2 --- /dev/null +++ b/tests/test_rename.py @@ -0,0 +1,21 @@ +from obstore.store import MemoryStore + + +def test_rename_non_ascii(): + store = MemoryStore() + + name1 = "café.txt" + name2 = "ümlaut.txt" + name3 = "こんにちは世界.txt" + name4 = "你好世界.txt" + + store.put(name1, b"foo") + store.put(name3, b"bar") + + store.rename(name1, name2) + store.rename(name3, name4) + + result = store.list().collect() + assert len(result) == 2 + assert result[0]["path"] == name2 + assert result[1]["path"] == name4