add support for .gz, .bz2 and .owl files. Update docs

Author: GregHanson

Cargo.lock

@@ -15,9 +15,11 @@ name = "benchmark"
 harness = false
 
 [dependencies]
+bzip2 = "0.6"
 clap = { version = "4.5", features = ["derive","cargo"] }
 clap-verbosity-flag = "3.0"
 env_logger = "0.11"
+flate2 = "1"
 hdt = { version = "0.6", default-features = false, features = ["nt"]}
 log = "0.4"
 oxrdf = "0.3"

Cargo.toml

@@ -12,6 +12,8 @@ and then generates and saves the data as HDT. Implementation is based on the [HD
 and the output HDT is intended to be consumed by one of [hdt crate](https://github.com/KonradHoeffner/hdt), [hdt-cpp](https://github.com/rdfhdt/hdt-cpp),
 or [hdt-java](https://github.com/rdfhdt/hdt-java).
 
+Inputs ending in `.gz` or `.bz2` are decompressed transparently, and `.owl` files are parsed as RDF/XML.
+
 ## Installation
 
 Install `rdf2hdt` with `cargo`:
@@ -36,7 +38,7 @@ Options:
   -i, --input <INPUT>...
           Path to input RDF file(s).
 
-          Provide the path to one or more RDF files that will be parsed and converted. Support file formats: https://crates.io/crates/oxrdfio
+          Provide the path to one or more RDF files that will be parsed and converted. RDF syntaxes supported: see https://crates.io/crates/oxrdfio. `.owl` files are parsed as RDF/XML. Inputs ending in `.gz` or `.bz2` are transparently decompressed.
 
   -o, --output <OUTPUT>
           Path to output file.

README.md

@@ -12,6 +12,8 @@
 //! ## Features
 //! - Parses RDF input and converts it to RDF triples
 //! - Convert NTriple data into HDT format
+//! - Transparent gzip (`.gz`) and bzip2 (`.bz2`) decompression of input files
+//! - `.owl` files parsed as RDF/XML
 //!
 //! ## Usage
 //! Run the rdf2hdt converter from the command line. For detailed usage information, run:
@@ -56,7 +58,9 @@ enum Commands {
         /// Path to input RDF file(s).
         ///
         /// Provide the path to one or more RDF files that will be parsed and converted.
-        /// Support file formats: https://crates.io/crates/oxrdfio
+        /// RDF syntaxes supported: see https://crates.io/crates/oxrdfio.
+        /// `.owl` files are parsed as RDF/XML. Inputs ending in `.gz` or `.bz2`
+        /// are transparently decompressed.
         #[arg(short, long, num_args = 1..)]
         input: Vec<String>,
 

src/main.rs

@@ -2,51 +2,86 @@
 // Licensed under the BSD 3-Clause License (see LICENSE file in the project root).
 
 use crate::builder::Error;
+use bzip2::bufread::MultiBzDecoder;
+use flate2::bufread::MultiGzDecoder;
 use log::{debug, error, warn};
 use oxrdfio::RdfSerializer;
 use oxrdfio::{
     RdfFormat::{self, NTriples},
     RdfParseError, RdfParser,
 };
+use std::fs::File;
 use std::io::Write;
 use std::{
-    io::{self, BufReader, BufWriter},
-    path::Path,
+    io::{self, BufReader, BufWriter, Read},
+    path::{Path, PathBuf},
 };
 use url::Url;
 
 const IO_BUF: usize = 1 << 20;
 
+fn open_rdf_reader(file: &Path) -> io::Result<Box<dyn Read>> {
+    let fp = File::open(file)?;
+    let buffered = BufReader::with_capacity(IO_BUF, fp);
+    match file.extension().and_then(|e| e.to_str()) {
+        Some(e) if e.eq_ignore_ascii_case("gz") => Ok(Box::new(BufReader::with_capacity(
+            IO_BUF,
+            MultiGzDecoder::new(buffered),
+        ))),
+        Some(e) if e.eq_ignore_ascii_case("bz2") => Ok(Box::new(BufReader::with_capacity(
+            IO_BUF,
+            MultiBzDecoder::new(buffered),
+        ))),
+        _ => Ok(Box::new(buffered)),
+    }
+}
+
+fn rdf_format_from_path(file: &Path) -> io::Result<RdfFormat> {
+    let format_path: PathBuf = match file.extension().and_then(|e| e.to_str()) {
+        Some(e) if e.eq_ignore_ascii_case("gz") || e.eq_ignore_ascii_case("bz2") => {
+            file.with_extension("")
+        }
+        _ => file.to_path_buf(),
+    };
+
+    let ext = format_path
+        .extension()
+        .and_then(|e| e.to_str())
+        .ok_or_else(|| {
+            io::Error::new(
+                io::ErrorKind::InvalidInput,
+                format!("file {} has no usable extension", file.display()),
+            )
+        })?;
+    if ext.eq_ignore_ascii_case("owl") {
+        return Ok(RdfFormat::RdfXml);
+    }
+    RdfFormat::from_extension(ext).ok_or_else(|| {
+        error!("unrecognized file extension for {}", file.display());
+        io::Error::new(
+            io::ErrorKind::InvalidData,
+            format!("unrecognized file extension for {}", file.display()),
+        )
+    })
+}
+
 pub(crate) fn convert_to_nt<P: AsRef<Path>>(
     file_paths: &[P],
     output_file: std::fs::File,
 ) -> Result<(), Error> {
     let mut dest_writer = BufWriter::with_capacity(IO_BUF, output_file);
     for file in file_paths {
         let file = file.as_ref();
-        let source = std::fs::File::open(file).map_err(|e| {
+        let source_reader = open_rdf_reader(file).map_err(|e| {
             error!("Error opening file {}: {e:?}", file.display());
             e
         })?;
-        let source_reader = BufReader::with_capacity(IO_BUF, source);
 
         debug!("converting {} to nt format", file.display());
 
         let mut serializer = RdfSerializer::from_format(NTriples).for_writer(dest_writer.by_ref());
         let v = std::time::Instant::now();
-        let ext = file.extension().and_then(|e| e.to_str()).ok_or_else(|| {
-            io::Error::new(
-                io::ErrorKind::InvalidInput,
-                format!("file {} has no usable extension", file.display()),
-            )
-        })?;
-        let rdf_format = RdfFormat::from_extension(ext).ok_or_else(|| {
-            error!("unrecognized file extension for {}", file.display());
-            io::Error::new(
-                io::ErrorKind::InvalidData,
-                format!("unrecognized file extension for {}", file.display()),
-            )
-        })?;
+        let rdf_format = rdf_format_from_path(file)?;
         let abs_path = std::fs::canonicalize(file)?;
         let base_iri = Url::from_file_path(&abs_path).map_err(|_| {
             io::Error::new(
@@ -133,4 +168,65 @@ mod tests {
         assert!(quads.is_ok());
         assert_eq!(quads.unwrap().len(), 9)
     }
+
+    fn count_triples_in(nt_file: &tempfile::NamedTempFile) -> usize {
+        let reader = BufReader::new(nt_file.reopen().expect("error opening tmp file"));
+        RdfParser::from_format(NTriples)
+            .for_reader(reader)
+            .collect::<Result<Vec<_>, _>>()
+            .expect("parse nt")
+            .len()
+    }
+
+    #[test]
+    fn gzipped_ttl_input() -> Result<(), Error> {
+        let tmp = tempfile::tempdir()?;
+        let gz_path = tmp.path().join("apple.ttl.gz");
+        let source = std::fs::read("tests/resources/apple.ttl")?;
+        let mut enc =
+            flate2::write::GzEncoder::new(File::create(&gz_path)?, flate2::Compression::default());
+        enc.write_all(&source)?;
+        enc.finish()?;
+
+        let out = tempfile::Builder::new().suffix(".nt").tempfile()?;
+        convert_to_nt(&[&gz_path], out.reopen()?)?;
+        assert_eq!(count_triples_in(&out), 9);
+        Ok(())
+    }
+
+    #[test]
+    fn bzipped_ttl_input() -> Result<(), Error> {
+        let tmp = tempfile::tempdir()?;
+        let bz_path = tmp.path().join("apple.ttl.bz2");
+        let source = std::fs::read("tests/resources/apple.ttl")?;
+        let mut enc =
+            bzip2::write::BzEncoder::new(File::create(&bz_path)?, bzip2::Compression::default());
+        enc.write_all(&source)?;
+        enc.finish()?;
+
+        let out = tempfile::Builder::new().suffix(".nt").tempfile()?;
+        convert_to_nt(&[&bz_path], out.reopen()?)?;
+        assert_eq!(count_triples_in(&out), 9);
+        Ok(())
+    }
+
+    #[test]
+    fn owl_as_rdfxml() -> Result<(), Error> {
+        let tmp = tempfile::tempdir()?;
+        let owl_path = tmp.path().join("tiny.owl");
+        std::fs::write(
+            &owl_path,
+            r#"<?xml version="1.0"?>
+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+  <rdf:Description rdf:about="http://example.org/a">
+    <rdf:type rdf:resource="http://example.org/Thing"/>
+  </rdf:Description>
+</rdf:RDF>"#,
+        )?;
+
+        let out = tempfile::Builder::new().suffix(".nt").tempfile()?;
+        convert_to_nt(&[&owl_path], out.reopen()?)?;
+        assert_eq!(count_triples_in(&out), 1);
+        Ok(())
+    }
 }