Skip to content

Commit

Permalink
issue-1292: sql projection test case
Browse files Browse the repository at this point in the history
This commit adds a unit test demonstrating the issue described
in delta-io#1292.
  • Loading branch information
cmackenzie1 committed Apr 16, 2023
1 parent d7eb60b commit 39c9eb3
Show file tree
Hide file tree
Showing 5 changed files with 35 additions and 2 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{"commitInfo":{"delta-rs":"0.9.0","timestamp":1681604880998}}
{"protocol":{"minReaderVersion":1,"minWriterVersion":1}}
{"metaData":{"id":"1c863a88-7231-4ffc-a24b-3bbfad87d80a","name":"http_requests","description":"","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"date\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ClientIP\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ClientRequestHost\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ClientRequestMethod\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ClientRequestURI\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"EdgeEndTimestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"EdgeResponseBytes\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"EdgeResponseStatus\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"EdgeStartTimestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["date"],"createdTime":1681604880998,"configuration":{}}}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{"add":{"path":"date=2023-04-14/part-00000-731ab1b3-85a8-4bc3-92e5-96347fe3fd84-c000.snappy.parquet","size":5976,"partitionValues":{"date":"2023-04-14"},"modificationTime":1681604881837,"dataChange":true,"stats":"{\"numRecords\":1437,\"minValues\":{\"EdgeEndTimestamp\":\"2023-04-14T00:00:00.000Z\",\"ClientRequestHost\":\"example.com\",\"EdgeResponseStatus\":200,\"ClientRequestMethod\":\"GET\",\"EdgeStartTimestamp\":\"2023-04-14T00:00:00.000Z\",\"ClientIP\":\"127.0.0.1\",\"ClientRequestURI\":\"/\",\"EdgeResponseBytes\":303},\"maxValues\":{\"EdgeResponseBytes\":307,\"ClientRequestURI\":\"/\",\"EdgeResponseStatus\":200,\"EdgeStartTimestamp\":\"2023-04-14T00:00:45.000Z\",\"ClientIP\":\"127.0.0.1\",\"ClientRequestHost\":\"example.com\",\"ClientRequestMethod\":\"GET\",\"EdgeEndTimestamp\":\"2023-04-14T00:00:45.000Z\"},\"nullCount\":{\"ClientRequestHost\":0,\"ClientRequestURI\":0,\"ClientRequestMethod\":0,\"EdgeResponseStatus\":0,\"ClientIP\":0,\"EdgeEndTimestamp\":0,\"EdgeResponseBytes\":0,\"EdgeStartTimestamp\":0}}","tags":null}}
{"add":{"path":"date=2023-04-13/part-00000-e853fe2e-6f42-450c-8af1-4145b73a96c7-c000.snappy.parquet","size":3780,"partitionValues":{"date":"2023-04-13"},"modificationTime":1681604881849,"dataChange":true,"stats":"{\"numRecords\":144,\"minValues\":{\"EdgeStartTimestamp\":\"2023-04-13T23:58:58.000Z\",\"ClientRequestMethod\":\"GET\",\"ClientIP\":\"127.0.0.1\",\"EdgeEndTimestamp\":\"2023-04-13T23:58:58.000Z\",\"ClientRequestURI\":\"/\",\"ClientRequestHost\":\"example.com\",\"EdgeResponseBytes\":303,\"EdgeResponseStatus\":200},\"maxValues\":{\"ClientRequestHost\":\"example.com\",\"ClientRequestURI\":\"/\",\"ClientIP\":\"127.0.0.1\",\"EdgeResponseStatus\":200,\"ClientRequestMethod\":\"GET\",\"EdgeStartTimestamp\":\"2023-04-13T23:59:59.000Z\",\"EdgeEndTimestamp\":\"2023-04-14T00:00:00.000Z\",\"EdgeResponseBytes\":307},\"nullCount\":{\"ClientRequestMethod\":0,\"ClientIP\":0,\"EdgeResponseStatus\":0,\"ClientRequestURI\":0,\"EdgeEndTimestamp\":0,\"EdgeStartTimestamp\":0,\"EdgeResponseBytes\":0,\"ClientRequestHost\":0}}","tags":null}}
{"commitInfo":{"timestamp":1681604881849,"clientVersion":"delta-rs.0.9.0"}}
Binary file not shown.
Binary file not shown.
31 changes: 29 additions & 2 deletions rust/tests/datafusion_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ use arrow::datatypes::{
use arrow::record_batch::RecordBatch;
use common::datafusion::context_with_delta_table_factory;
use datafusion::assert_batches_sorted_eq;
use datafusion::datasource::TableProvider;
use datafusion::datasource::{TableProvider};
use datafusion::execution::context::{SessionContext, SessionState, TaskContext};
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion::physical_plan::{common::collect, file_format::ParquetExec, metrics::Label};
use datafusion::physical_plan::{visit_execution_plan, ExecutionPlan, ExecutionPlanVisitor};
use datafusion_common::scalar::ScalarValue;
use datafusion_common::ScalarValue::*;
use datafusion_common::{DataFusionError, Result};
use datafusion_expr::Expr;
use datafusion_expr::{Expr};
use datafusion_proto::bytes::{
physical_plan_from_bytes_with_extension_codec, physical_plan_to_bytes_with_extension_codec,
};
Expand Down Expand Up @@ -763,3 +763,30 @@ async fn test_datafusion_scan_timestamps() -> Result<()> {

Ok(())
}

#[tokio::test]
async fn test_datafusion_sql_projection() -> Result<()> {
let ctx = SessionContext::new();
let table = deltalake::open_table("./tests/data/http_requests")
.await
.unwrap();
ctx.register_table("http_requests", Arc::new(table))?;

let batches = ctx.sql("SELECT \"ClientRequestURI\" FROM http_requests LIMIT 5").await?.collect().await?;

let expected = vec![
"+------------------+",
"| ClientRequestURI |",
"+------------------+",
"| / |",
"| / |",
"| / |",
"| / |",
"| / |",
"+------------------+",
];

assert_batches_sorted_eq!(&expected, &batches);

Ok(())
}

0 comments on commit 39c9eb3

Please sign in to comment.