add w3c/rdf-tests as submodule. fixes for found conversion failures

Author: GregHanson

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/resources/rdf-tests"]
+	path = tests/resources/rdf-tests
+	url = https://github.com/w3c/rdf-tests.git

Cargo.lock

@@ -15,8 +15,7 @@ name = "benchmark"
 harness = false
 
 [dependencies]
-anyhow = "1.0.98"
-chrono = "0.4.40"
+chrono = "0.4"
 clap = { version = "4.5", features = ["derive","cargo"] }
 clap-verbosity-flag = "3.0"
 crc = "3.2.1"
@@ -30,6 +29,7 @@ tempfile = "3.19"
 [dev-dependencies]
 criterion = { version = "0.5", features = ["async_tokio"]}
 pprof = { version = "0.14", features = ["protobuf", "protobuf-codec", "criterion"] }
+walkdir = "2.5"
 
 [package.metadata.cargo-machete]
-ignored = ["iref", "langtag", "hdt"]
+ignored = ["iref", "langtag"]

Cargo.toml

@@ -121,7 +121,11 @@ impl BitmapTriplesBuilder {
         // bitmapZ->save(output);
         self.save_bitmap(&self.bitmap_z, dest_writer)?;
 
-        let num_bits = self.num_triples.ilog2() + 1;
+        let num_bits = if self.num_triples == 0 {
+            0
+        } else {
+            self.num_triples.ilog2() + 1
+        };
         if num_bits > u8::MAX as u32 {
             panic!("bits_per_entry too large")
         }

src/bitmap_triples.rs

@@ -282,6 +282,8 @@ mod tests {
         io::{BufReader, Read},
         path::Path,
     };
+    use tempfile::tempdir;
+    use walkdir::WalkDir;
 
     #[test]
     fn test_build_hdt() {
@@ -343,4 +345,90 @@ mod tests {
             .collect::<Vec<_>>();
         assert_eq!(triple_vec, res)
     }
+
+    #[test]
+    fn test_sparql10() {
+        let tmp_dir = tempdir().expect("failed to create temp dir");
+        let input_files = find_ttl_files("tests/resources/rdf-tests/sparql/sparql10");
+        for f in &input_files {
+            let new_hdt = format!(
+                "{}/{}",
+                tmp_dir.path().to_str().unwrap(),
+                Path::new(f)
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .replace(".ttl", ".hdt")
+            );
+
+            let res = build_hdt(vec![f.clone()], new_hdt.as_str(), Options::default());
+            assert!(res.is_ok());
+        }
+    }
+
+    #[test]
+    fn test_sparql11() {
+        let tmp_dir = tempdir().expect("failed to create temp dir");
+        let input_files = find_ttl_files("tests/resources/rdf-tests/sparql/sparql11");
+        for f in &input_files {
+            let new_hdt = format!(
+                "{}/{}",
+                tmp_dir.path().to_str().unwrap(),
+                Path::new(f)
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .replace(".ttl", ".hdt")
+            );
+
+            let res = build_hdt(vec![f.clone()], new_hdt.as_str(), Options::default());
+            match &res {
+                Ok(_) => {}
+                Err(e) => {
+                    eprintln!("{f} failed to convert: {e}")
+                }
+            }
+
+            assert!(res.is_ok());
+        }
+    }
+
+    #[test]
+    fn test_sparql12() {
+        let tmp_dir = tempdir().expect("failed to create temp dir");
+        let input_files = find_ttl_files("tests/resources/rdf-tests/sparql/sparql12");
+        for f in &input_files {
+            let new_hdt = format!(
+                "{}/{}",
+                tmp_dir.path().to_str().unwrap(),
+                Path::new(f)
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .replace(".ttl", ".hdt")
+            );
+
+            let res = build_hdt(vec![f.clone()], new_hdt.as_str(), Options::default());
+            match &res {
+                Ok(_) => {}
+                Err(e) => {
+                    eprintln!("{f} failed to convert: {e}")
+                }
+            }
+
+            assert!(res.is_ok());
+        }
+    }
+
+    fn find_ttl_files<P: AsRef<Path>>(dir: P) -> Vec<String> {
+        WalkDir::new(dir)
+            .into_iter()
+            .filter_map(|e| e.ok())
+            .filter(|e| e.path().extension().is_some_and(|ext| ext == "ttl"))
+            .map(|e| e.path().display().to_string())
+            .collect()
+    }
 }

src/builder.rs

@@ -46,8 +46,6 @@ pub fn save_u32_vec(
 
 // TODO duplicate of containers/sequence.rs::save()
 fn pack_bits(data: &[u32], bits_per_entry: u8) -> Vec<u8> {
-    assert!(bits_per_entry > 0 && bits_per_entry as usize <= std::mem::size_of::<usize>() * 8);
-
     let mut output = Vec::new();
     let mut current_byte = 0u8;
     let mut bit_offset = 0;

src/common.rs

@@ -5,7 +5,7 @@ use hdt::{
     containers::{self, ControlType, Sequence, vbyte::encode_vbyte},
     dict_sect_pfc::DictSectPFC,
 };
-use log::{debug, error};
+use log::{debug, error, warn};
 use oxrdf::Term;
 use oxrdfio::RdfFormat::NTriples;
 use oxrdfio::RdfParser;
@@ -90,8 +90,7 @@ impl FourSectDictBuilder {
             object_terms.insert(term_to_hdt_bgp_str(&q.object)?);
         }
         if dict.predicate_terms.is_empty() {
-            error!("no triples found in provided RDF");
-            return Err(anyhow::anyhow!("no triples found in input file").into());
+            warn!("no triples found in provided RDF");
         }
 
         dict.shared_terms = subject_terms.intersection(&object_terms).cloned().collect();
@@ -282,7 +281,11 @@ pub fn compress(set: &BTreeSet<String>, block_size: usize) -> Result<DictSectPFC
 
     // offsets are an increasing list of array indices, therefore the last one will be the largest
     // TODO: potential off by 1 in comparison with hdt-cpp implementation?
-    let bits_per_entry = (offsets.last().unwrap().ilog2() + 1) as usize;
+    let bits_per_entry = if num_terms == 0 {
+        0
+    } else {
+        (offsets.last().unwrap().ilog2() + 1) as usize
+    };
 
     Ok(DictSectPFC {
         num_strings: num_terms,

src/dictionary.rs

@@ -45,7 +45,12 @@ pub fn convert_to_nt(
             )
             .into());
         };
-        let quads = RdfParser::from_format(rdf_format).for_reader(source_reader);
+        let quads = RdfParser::from_format(rdf_format)
+            .with_base_iri(format!(
+                "file://{}",
+                Path::new(&file).file_name().unwrap().to_str().unwrap()
+            ))?
+            .for_reader(source_reader);
         let mut warned = false;
         for q in quads {
             let q = match q {