hipvs/ivf_pq/
mod.rs

1/*
2 * Copyright (c) 2024, NVIDIA CORPORATION.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16//! Inverted File Product Quantization
17//!
18//! Example:
19//! ```
20//!
21//! use hipvs::ivf_pq::{Index, IndexParams, SearchParams};
22//! use hipvs::{ManagedTensor, Resources, Result};
23//!
24//! use ndarray::s;
25//! use ndarray_rand::rand_distr::Uniform;
26//! use ndarray_rand::RandomExt;
27//!
28//! fn ivf_pq_example() -> Result<()> {
29//!     let res = Resources::new()?;
30//!
31//!     // Create a new random dataset to index
32//!     let n_datapoints = 65536;
33//!     let n_features = 512;
34//!     let dataset =
35//!         ndarray::Array::<f32, _>::random((n_datapoints, n_features), Uniform::new(0., 1.0));
36//!
37//!     // build the ivf-pq index
38//!     let build_params = IndexParams::new()?;
39//!     let index = Index::build(&res, &build_params, &dataset)?;
40//!     println!(
41//!         "Indexed {}x{} datapoints into ivf-pq index",
42//!         n_datapoints, n_features
43//!     );
44//!
45//!     // use the first 4 points from the dataset as queries : will test that we get them back
46//!     // as their own nearest neighbor
47//!     let n_queries = 4;
48//!     let queries = dataset.slice(s![0..n_queries, ..]);
49//!
50//!     let k = 10;
51//!
52//!     // Ivf-Pq search API requires queries and outputs to be on device memory
53//!     // copy query data over, and allocate new device memory for the distances/ neighbors
54//!     // outputs
55//!     let queries = ManagedTensor::from(&queries).to_device(&res)?;
56//!     let mut neighbors_host = ndarray::Array::<u32, _>::zeros((n_queries, k));
57//!     let neighbors = ManagedTensor::from(&neighbors_host).to_device(&res)?;
58//!
59//!     let mut distances_host = ndarray::Array::<f32, _>::zeros((n_queries, k));
60//!     let distances = ManagedTensor::from(&distances_host).to_device(&res)?;
61//!
62//!     let search_params = SearchParams::new()?;
63//!
64//!     index.search(&res, &search_params, &queries, &neighbors, &distances)?;
65//!
66//!     // Copy back to host memory
67//!     distances.to_host(&res, &mut distances_host)?;
68//!     neighbors.to_host(&res, &mut neighbors_host)?;
69//!
70//!     // nearest neighbors should be themselves, since queries are from the
71//!     // dataset
72//!     println!("Neighbors {:?}", neighbors_host);
73//!     println!("Distances {:?}", distances_host);
74//!     Ok(())
75//! }
76//! ```
77
78mod index;
79mod index_params;
80mod search_params;
81
82pub use index::Index;
83pub use index_params::IndexParams;
84pub use search_params::SearchParams;
hipvs/ivf_pq/mod.rs

hipvs/ivf_pq/
mod.rs
