File scanning not done properly on date type, it scans all files. #220
Description
My query is this
query = f"""
SELECT count(*)
FROM delta_scan('{abfs_path}')
where date between '2025-05-02'AND and '2025-05-12'
"""And I even generated the profiling file
{
"result_set_size": 8,
"cumulative_rows_scanned": 15338000,
"cpu_time": 3.7344108279999992,
"system_peak_buffer_memory": 1368096,
"extra_info": {},
"blocked_thread_time": 0.0,
"rows_returned": 1,
"system_peak_temp_dir_size": 0,
"cumulative_cardinality": 245442,
"query_name": "\n SELECT count(*)\n FROM delta_scan('abfss://delta-path')\n where date between '2025-05-02' and '2025-05-12'\n",
"latency": 2.564646083,
"children": [
{
"result_set_size": 8,
"operator_timing": 0.00003220200000000001,
"operator_rows_scanned": 0,
"cumulative_rows_scanned": 15338000,
"operator_cardinality": 1,
"operator_type": "UNGROUPED_AGGREGATE",
"operator_name": "UNGROUPED_AGGREGATE",
"cpu_time": 3.7344108279999992,
"extra_info": {
"Aggregates": "count_star()"
},
"cumulative_cardinality": 245442,
"children": [
{
"cumulative_cardinality": 245441,
"extra_info": {
"Projections": "",
"Filters": "date>='2025-05-02'::DATE AND date<='2025-05-12'::DATE",
"File Filters": "date>='2025-05-02'::DATE AND date<='2025-05-12'::DATE",
"Scanning Files": "67/67",
"Estimated Cardinality": "383450",
"Total Files Read": "67"
},
"operator_name": "DELTA_SCAN ",
"cpu_time": 3.7343786259999994,
"operator_type": "TABLE_SCAN",
"operator_cardinality": 245441,
"cumulative_rows_scanned": 15338000,
"operator_rows_scanned": 15338000,
"operator_timing": 3.7343786259999994,
"result_set_size": 981764,
"children": []
}
]
}
]
}It says all 67/67 partitions scanned
whereas it should have only scanned some 10-11 partitions.
Below are how my partitions are.
Expected is, that it only scans 10-12 partitions, as the date range suggests.